diff --git a/.claude/commands/apple-container.md b/.claude/commands/apple-container.md new file mode 100644 index 00000000000..ca6876ad530 --- /dev/null +++ b/.claude/commands/apple-container.md @@ -0,0 +1,46 @@ +--- +allowed-tools: Bash(container *), Bash(cargo *), Read, Grep, Glob +--- + +# Run Tests in Linux Container (Apple `container` CLI) + +Run RustPython tests inside a Linux container using Apple's `container` CLI. +**NEVER use Docker, Podman, or any other container runtime.** Only use the `container` command. + +## Arguments +- `$ARGUMENTS`: Test command to run (e.g., `test_io`, `test_codecs -v`, `test_io -v -m "test_errors"`) + +## Prerequisites + +The `container` CLI is installed via `brew install container`. +The dev image `rustpython-dev` is already built. + +## Steps + +1. **Check if the container is already running** + ```shell + container list 2>/dev/null | grep rustpython-test + ``` + +2. **Start the container if not running** + ```shell + container run -d --name rustpython-test -m 8G -c 4 \ + --mount type=bind,source=/Users/al03219714/Projects/RustPython3,target=/workspace \ + -w /workspace rustpython-dev sleep infinity + ``` + +3. **Run the test inside the container** + ```shell + container exec rustpython-test sh -c "cargo run --release -- -m test $ARGUMENTS" + ``` + +4. **Report results** + - Show test summary (pass/fail counts, expected failures, unexpected successes) + - Highlight any new failures compared to macOS results if available + - Do NOT stop or remove the container after testing (keep it for reuse) + +## Notes +- The workspace is bind-mounted, so local code changes are immediately available +- Use `container exec rustpython-test sh -c "..."` for any command inside the container +- To rebuild after code changes, run: `container exec rustpython-test sh -c "cargo build --release"` +- To stop the container when done: `container rm -f rustpython-test` diff --git a/.claude/commands/investigate-test-failure.md b/.claude/commands/investigate-test-failure.md new file mode 100644 index 00000000000..e9d6b2d2d2c --- /dev/null +++ b/.claude/commands/investigate-test-failure.md @@ -0,0 +1,49 @@ +--- +allowed-tools: Bash(python3:*), Bash(cargo run:*), Read, Grep, Glob, Bash(git add:*), Bash(git commit:*), Bash(cargo fmt:*), Bash(git diff:*), Task +--- + +# Investigate Test Failure + +Investigate why a specific test is failing and determine if it can be fixed or needs an issue. + +## Arguments +- `$ARGUMENTS`: Failed test identifier (e.g., `test_inspect.TestGetSourceBase.test_getsource_reload`) + +## Steps + +1. **Analyze failure cause** + - Read the test code + - Analyze failure message/traceback + - Check related RustPython code + +2. **Verify behavior in CPython** + - Run the test with `python3 -m unittest` to confirm expected behavior + - Document the expected output + +3. **Determine fix feasibility** + - **Simple fix** (import issues, small logic bugs): Fix code → Run `cargo fmt --all` → Pre-commit review → Commit + - **Complex fix** (major unimplemented features): Collect issue info and report to user + + **Pre-commit review process**: + - Run `git diff` to see the changes + - Use Task tool with `general-purpose` subagent to review: + - Compare implementation against cpython/ source code + - Verify the fix aligns with CPython behavior + - Check for any missed edge cases + - Proceed to commit only after review passes + +4. **For complex issues - Collect issue information** + Following `.github/ISSUE_TEMPLATE/report-incompatibility.md` format: + + - **Feature**: Description of missing/broken Python feature + - **Minimal reproduction code**: Smallest code that reproduces the issue + - **CPython behavior**: Result when running with python3 + - **RustPython behavior**: Result when running with cargo run + - **Python Documentation link**: Link to relevant CPython docs + + Report collected information to the user. Issue creation is done only upon user request. + + Example issue creation command: + ``` + gh issue create --template report-incompatibility.md --title "..." --body "..." + ``` diff --git a/.claude/commands/upgrade-pylib-next.md b/.claude/commands/upgrade-pylib-next.md new file mode 100644 index 00000000000..712b79433b3 --- /dev/null +++ b/.claude/commands/upgrade-pylib-next.md @@ -0,0 +1,33 @@ +--- +allowed-tools: Skill(upgrade-pylib), Bash(gh pr list:*) +--- + +# Upgrade Next Python Library + +Find the next Python library module ready for upgrade and run `/upgrade-pylib` for it. + +## Current TODO Status + +!`cargo run --release -- scripts/update_lib todo 2>/dev/null` + +## Open Upgrade PRs + +!`gh pr list --search "Update in:title" --json number,title --template '{{range .}}#{{.number}} {{.title}}{{"\n"}}{{end}}'` + +## Instructions + +From the TODO list above, find modules matching these patterns (in priority order): + +1. `[ ] [no deps]` - Modules with no dependencies (can be upgraded immediately) +2. `[ ] [0/n]` - Modules where all dependencies are already upgraded (e.g., `[0/3]`, `[0/5]`) + +These patterns indicate modules that are ready to upgrade without blocking dependencies. + +**Important**: Skip any modules that already have an open PR in the "Open Upgrade PRs" list above. + +**After identifying a suitable module**, run: +``` +/upgrade-pylib +``` + +If no modules match these criteria, inform the user that all eligible modules have dependencies that need to be upgraded first. diff --git a/.claude/commands/upgrade-pylib.md b/.claude/commands/upgrade-pylib.md index ba4cef525ab..d54305d2616 100644 --- a/.claude/commands/upgrade-pylib.md +++ b/.claude/commands/upgrade-pylib.md @@ -1,3 +1,7 @@ +--- +allowed-tools: Bash(git add:*), Bash(git commit:*), Bash(python3 scripts/update_lib quick:*), Bash(python3 scripts/update_lib auto-mark:*) +--- + # Upgrade Python Library from CPython Upgrade a Python standard library module from CPython to RustPython. @@ -23,24 +27,19 @@ This helps improve the tooling for future upgrades. ## Steps -1. **Delete existing library in Lib/** - - If `Lib/$ARGUMENTS.py` exists, delete it - - If `Lib/$ARGUMENTS/` directory exists, delete it - -2. **Copy from cpython/Lib/** - - If `cpython/Lib/$ARGUMENTS.py` exists, copy it to `Lib/$ARGUMENTS.py` - - If `cpython/Lib/$ARGUMENTS/` directory exists, copy it to `Lib/$ARGUMENTS/` - -3. **Upgrade tests (quick upgrade with update_lib)** - - Run: `python3 scripts/update_lib quick cpython/Lib/test/test_$ARGUMENTS.py` (single file) - - Or: `python3 scripts/update_lib quick cpython/Lib/test/test_$ARGUMENTS/` (directory) +1. **Run quick upgrade with update_lib** + - Run: `python3 scripts/update_lib quick $ARGUMENTS` (module name) + - Or: `python3 scripts/update_lib quick cpython/Lib/$ARGUMENTS.py` (library file path) + - Or: `python3 scripts/update_lib quick cpython/Lib/$ARGUMENTS/` (library directory path) - This will: + - Copy library files (delete existing `Lib/$ARGUMENTS.py` or `Lib/$ARGUMENTS/`, then copy from `cpython/Lib/`) - Patch test files preserving existing RustPython markers - Run tests and auto-mark new test failures (not regressions) - Remove `@unittest.expectedFailure` from tests that now pass - - **Handle warnings**: If you see warnings like `WARNING: TestCFoo does not exist in remote file`, it means the class structure changed and markers couldn't be transferred automatically. These need to be manually restored in step 4 or added in step 5. + - Create a git commit with the changes + - **Handle warnings**: If you see warnings like `WARNING: TestCFoo does not exist in remote file`, it means the class structure changed and markers couldn't be transferred automatically. These need to be manually restored in step 2 or added in step 3. -4. **Review git diff and restore RUSTPYTHON-specific changes** +2. **Review git diff and restore RUSTPYTHON-specific changes** - Run `git diff Lib/test/test_$ARGUMENTS` to review all changes - **Only restore changes that have explicit `RUSTPYTHON` comments**. Look for: - `# XXX: RUSTPYTHON` or `# XXX RUSTPYTHON` - Comments marking RustPython-specific code modifications @@ -49,13 +48,46 @@ This helps improve the tooling for future upgrades. - **Do NOT restore other diff changes** - these are likely upstream CPython changes, not RustPython-specific modifications - When restoring, preserve the original context and formatting -5. **Verify tests** - - Run: `cargo run --release -- -m test test_$ARGUMENTS -v` - - The `-v` flag shows detailed output to identify which tests fail and why - - For each new failure, add appropriate markers based on the failure type: - - **Test assertion failure** → `@unittest.expectedFailure` with `# TODO: RUSTPYTHON` comment - - **Panic/crash** → `@unittest.skip("TODO: RUSTPYTHON; ")` - - **Class-specific markers**: If a test fails only in the C implementation (TestCFoo) but passes in the Python implementation (TestPyFoo), or vice versa, add the marker to the specific subclass, not the base class: +3. **Investigate test failures with subagent** + - First, get dependent tests using the deps command: + ``` + cargo run --release -- scripts/update_lib deps $ARGUMENTS + ``` + - Look for the line `- [ ] $ARGUMENTS: test_xxx test_yyy ...` to get the direct dependent tests + - Run those tests to collect failures: + ``` + cargo run --release -- -m test test_xxx test_yyy ... 2>&1 | grep -E "^(FAIL|ERROR):" + ``` + - For example, if deps output shows `- [ ] linecache: test_bdb test_inspect test_linecache test_traceback test_zipimport`, run: + ``` + cargo run --release -- -m test test_bdb test_inspect test_linecache test_traceback test_zipimport 2>&1 | grep -E "^(FAIL|ERROR):" + ``` + - For each failure, use the Task tool with `general-purpose` subagent to investigate: + - Subagent should follow the `/investigate-test-failure` skill workflow + - Pass the failed test identifier as the argument (e.g., `test_inspect.TestGetSourceBase.test_getsource_reload`) + - If subagent can fix the issue easily: fix and commit + - If complex issue: subagent collects issue info and reports back (issue creation on user request only) + - Using subagent prevents context pollution in the main conversation + +4. **Mark remaining test failures with auto-mark** + - Run: `python3 scripts/update_lib auto-mark Lib/test/test_$ARGUMENTS.py --mark-failure` + - Or for directory: `python3 scripts/update_lib auto-mark Lib/test/test_$ARGUMENTS/ --mark-failure` + - This will: + - Run tests and mark ALL failing tests with `@unittest.expectedFailure` + - Remove `@unittest.expectedFailure` from tests that now pass + - **Note**: The `--mark-failure` flag marks all failures including regressions. Review the changes before committing. + +5. **Handle panics manually** + - If any tests cause panics/crashes (not just assertion failures), they need `@unittest.skip` instead: + ```python + @unittest.skip("TODO: RUSTPYTHON; panics with 'index out of bounds'") + def test_crashes(self): + ... + ``` + - auto-mark cannot detect panics automatically - check the test output for crash messages + +6. **Handle class-specific failures** + - If a test fails only in the C implementation (TestCFoo) but passes in the Python implementation (TestPyFoo), or vice versa, move the marker to the specific subclass: ```python # Base class - no marker here class TestFoo: @@ -70,25 +102,21 @@ This helps improve the tooling for future upgrades. def test_something(self): return super().test_something() ``` - - **New tests from CPython**: The upgrade may bring in entirely new tests that didn't exist before. These won't have any RUSTPYTHON markers in the diff - they just need to be tested and marked if they fail. - - Example markers: - ```python - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_something(self): - ... - # TODO: RUSTPYTHON - @unittest.skip("TODO: RUSTPYTHON; panics with 'index out of bounds'") - def test_crashes(self): - ... - ``` +7. **Commit the test fixes** + - Run: `git add -u && git commit -m "Mark failing tests"` + - This creates a separate commit for the test markers added in steps 2-6 ## Example Usage ``` +# Using module names (recommended) /upgrade-pylib inspect /upgrade-pylib json /upgrade-pylib asyncio + +# Using library paths (alternative) +/upgrade-pylib cpython/Lib/inspect.py +/upgrade-pylib cpython/Lib/json/ ``` ## Example: Restoring RUSTPYTHON changes diff --git a/.claude/scripts/setup-env.sh b/.claude/scripts/setup-env.sh new file mode 100755 index 00000000000..f5d28a3f14c --- /dev/null +++ b/.claude/scripts/setup-env.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# Claude Code web session startup script +# Sets up the development environment for RustPython + +set -e + +cd /home/user/RustPython + +echo "=== RustPython dev environment setup ===" + +# 1. Ensure python3 points to 3.13+ (needed for scripts/update_lib) +# /usr/local/bin takes precedence over /usr/bin in PATH, +# so we update the symlink there directly. +CURRENT_PY=$(python3 --version 2>&1 | grep -oP '\d+\.\d+') +if [ "$(printf '%s\n' "3.13" "$CURRENT_PY" | sort -V | head -1)" != "3.13" ]; then + echo "Upgrading python3 default to 3.13..." + # Find best available Python >= 3.13 + TARGET="" + for ver in python3.14 python3.13; do + if command -v "$ver" &>/dev/null; then + TARGET=$(command -v "$ver") + break + fi + done + if [ -n "$TARGET" ]; then + # Override /usr/local/bin/python3 if it exists and is outdated + if [ -e /usr/local/bin/python3 ]; then + sudo ln -sf "$TARGET" /usr/local/bin/python3 + fi + # Also set /usr/bin via update-alternatives + sudo update-alternatives --install /usr/bin/python3 python3 "$TARGET" 3 2>/dev/null || true + sudo update-alternatives --set python3 "$TARGET" 2>/dev/null || true + echo "python3 now: $(python3 --version)" + else + echo "WARNING: No Python 3.13+ found. scripts/update_lib may not work." + fi +else + echo "python3 already >= 3.13: $(python3 --version)" +fi + +# 2. Clone CPython source if not present (needed for scripts/update_lib) +if [ ! -d "cpython" ]; then + echo "Cloning CPython v3.14.3 (shallow)..." + git clone --depth 1 --branch v3.14.3 https://github.com/python/cpython.git cpython + echo "CPython source ready." +else + echo "CPython source already present." +fi + +echo "=== Setup complete ===" diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 00000000000..22f0a9a8a01 --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,15 @@ +{ + "hooks": { + "SessionStart": [ + { + "matcher": "", + "hooks": [ + { + "type": "command", + "command": "bash .claude/scripts/setup-env.sh" + } + ] + } + ] + } +} diff --git a/.cspell.dict/cpython.txt b/.cspell.dict/cpython.txt index 7574d45c5f1..38e8656c3b2 100644 --- a/.cspell.dict/cpython.txt +++ b/.cspell.dict/cpython.txt @@ -10,7 +10,10 @@ badcert badsyntax baseinfo basetype +binop +bltin boolop +BUFMAX BUILDSTDLIB bxor byteswap @@ -25,18 +28,25 @@ carg cellarg cellvar cellvars +ceval cfield CLASSDEREF classdict cmpop codedepth CODEUNIT +CONIN +CONOUT +constevaluator +consti CONVFUNC convparam copyslot cpucount +datastack defaultdict denom +deopt dictbytype DICTFLAG dictoffset @@ -47,6 +57,8 @@ eofs evalloop excepthandler exceptiontable +fastlocal +fastlocals fblock fblocks fdescr @@ -55,8 +67,11 @@ fielddesc fieldlist fileutils finalbody +finalizers +firsttraceable flowgraph formatfloat +freelist freevar freevars fromlist @@ -72,6 +87,7 @@ HASPOINTER HASSTRUCT HASUNION heaptype +hexdigit HIGHRES IFUNC IMMUTABLETYPE @@ -93,6 +109,7 @@ lineiterator linetable loadfast localsplus +localspluskinds Lshift lsprof MAXBLOCKS @@ -101,7 +118,9 @@ metavars miscompiles mult multibytecodec +nameobj nameop +ncells nconsts newargs newfree @@ -110,6 +129,7 @@ newsemlockobject nfrees nkwargs nkwelts +nlocalsplus Nondescriptor noninteger nops @@ -135,11 +155,14 @@ prec preinitialized pybuilddir pycore +pyinner pydecimal Pyfunc +pylifecycle pymain pyrepl PYTHONTRACEMALLOC +PYTHONUTF8 pythonw PYTHREAD_NAME releasebuffer @@ -147,16 +170,21 @@ repr resinfo Rshift SA_ONSTACK +saveall scls setdict setfunc +setprofileallthreads SETREF setresult setslice +settraceallthreads SLOTDEFINED +SMALLBUF SOABI SSLEOF stackdepth +stackref staticbase stginfo storefast @@ -167,10 +195,14 @@ subparams subscr sval swappedbytes +sysdict templatelib +testconsole +threadstate ticketer tmptype tok_oldval +tstate tvars typeobject typeparam @@ -178,9 +210,11 @@ Typeparam typeparams typeslots unaryop +uncollectable Unhandle unparse unparser +untracking VARKEYWORDS varkwarg venvlauncher @@ -190,8 +224,11 @@ venvwlauncher venvwlaunchert wbits weakreflist +weakrefobject webpki +winconsoleio withitem withs +worklist xstat XXPRIME diff --git a/.cspell.dict/python-more.txt b/.cspell.dict/python-more.txt index 2dd31f8f579..2ce5d246d72 100644 --- a/.cspell.dict/python-more.txt +++ b/.cspell.dict/python-more.txt @@ -1,8 +1,10 @@ abiflags abstractmethods +addcompare aenter aexit aiter +altzone anext anextawaitable annotationlib @@ -24,6 +26,7 @@ breakpointhook cformat chunksize classcell +classmethods closefd closesocket codepoint @@ -32,6 +35,8 @@ codesize contextvar cpython cratio +ctype +ctypes dealloc debugbuild decompressor @@ -74,6 +79,8 @@ fstring fstrings ftruncate genexpr +genexpressions +getargs getattro getcodesize getdefaultencoding @@ -83,14 +90,17 @@ getformat getframe getframemodulename getnewargs +getopt getpip getrandom getrecursionlimit getrefcount getsizeof getswitchinterval +getweakref getweakrefcount getweakrefs +getweakrefs getwindowsversion gmtoff groupdict @@ -103,8 +113,12 @@ idxs impls indexgroup infj +inittab +Inittab instancecheck instanceof +interpchannels +interpqueues irepeat isabstractmethod isbytes @@ -129,6 +143,7 @@ listcomp longrange lvalue mappingproxy +markupbase maskpri maxdigits MAXGROUPS @@ -144,6 +159,7 @@ mformat mro mros multiarch +mymodule namereplace nanj nbytes @@ -156,6 +172,7 @@ nlocals NOARGS nonbytes Nonprintable +onceregistry origname ospath pendingcr @@ -170,7 +187,10 @@ profilefunc pycache pycodecs pycs +pydatetime pyexpat +pyio +pymain PYTHONAPI PYTHONBREAKPOINT PYTHONDEBUG @@ -220,10 +240,13 @@ scproxy seennl setattro setcomp +setprofileallthreads setrecursionlimit setswitchinterval +settraceallthreads showwarnmsg signum +sitebuiltins slotnames STACKLESS stacklevel @@ -232,14 +255,17 @@ startpos subclassable subclasscheck subclasshook +subclassing suboffset suboffsets SUBPATTERN +subpatterns sumprod surrogateescape surrogatepass sysconf sysconfigdata +sysdict sysvars teedata thisclass @@ -266,35 +292,10 @@ warnopts weaklist weakproxy weakrefs +weakrefset winver withdata xmlcharrefreplace xoptions xopts yieldfrom -addcompare -altzone -classmethods -ctype -ctypes -genexpressions -getargs -getopt -getweakref -getweakrefs -inittab -Inittab -interpchannels -interpqueues -markupbase -mymodule -pydatetime -pyio -pymain -setprofileallthreads -settraceallthreads -sitebuiltins -subclassing -subpatterns -sysdict -weakrefset diff --git a/.cspell.dict/rust-more.txt b/.cspell.dict/rust-more.txt index c3ebd61833a..c4457723c6c 100644 --- a/.cspell.dict/rust-more.txt +++ b/.cspell.dict/rust-more.txt @@ -5,7 +5,9 @@ biguint bindgen bitand bitflags +bitflagset bitor +bitvec bitxor bstr byteorder @@ -58,6 +60,7 @@ powi prepended punct replacen +retag rmatch rposition rsplitn @@ -89,5 +92,3 @@ widestring winapi winresource winsock -bitvec -Bitvec diff --git a/.cspell.dict/rustpython.txt b/.cspell.dict/rustpython.txt new file mode 100644 index 00000000000..8cd08358019 --- /dev/null +++ b/.cspell.dict/rustpython.txt @@ -0,0 +1,32 @@ +cfgs +miri +py +pyarg +pyargs +pyast +pyattr +pyclass +pyclassmethod +pyexception +pyfunction +pygetset +pyimpl +pylib +pymath +pymethod +pymodule +pyname +pyobj +pyobject +pypayload +pyref +pyslot +pystaticmethod +pystone +pystr +pystruct +pystructseq +pytype +rustix +struc +zelf diff --git a/.cspell.json b/.cspell.json index ebed8664e58..b7efb2e2311 100644 --- a/.cspell.json +++ b/.cspell.json @@ -9,6 +9,7 @@ "@cspell/dict-win32/cspell-ext.json", "@cspell/dict-shell/cspell-ext.json", ], + "allowCompoundWords": true, // language - current active spelling language "language": "en", // dictionaries - list of the names of the dictionaries to use @@ -16,6 +17,7 @@ "cpython", // Sometimes keeping same terms with cpython is easy "python-more", // Python API terms not listed in python "rust-more", // Rust API terms not listed in rust + "rustpython", // RustPython derive macros and internal terms "en_US", "softwareTerms", "c", @@ -38,6 +40,10 @@ { "name": "rust-more", "path": "./.cspell.dict/rust-more.txt" + }, + { + "name": "rustpython", + "path": "./.cspell.dict/rustpython.txt" } ], "ignorePaths": [ @@ -46,108 +52,42 @@ "Lib/**" ], // words - list of words to be always considered correct + // (compound words like pyarg, baseclass, microbenchmark are handled by allowCompoundWords) "words": [ - "RUSTPYTHONPATH", - // RustPython terms "aiterable", "alnum", - "baseclass", - "boxvec", - "Bytecode", - "cfgs", - "codegen", "coro", "dedentations", "dedents", "deduped", - "downcastable", - "downcasted", - "dumpable", + "deoptimize", "emscripten", "excs", - "finalizer", - "GetSet", - "groupref", - "internable", "interps", "jitted", "jitting", + "kwonly", "lossily", - "makeunicodedata", - "microbenchmark", - "microbenchmarks", - "miri", - "notrace", + "mcache", "oparg", - "openat", - "pyarg", - "pyargs", - "pyast", - "PyAttr", + "opargs", "pyc", - "PyClass", - "PyClassMethod", - "PyException", - "PyFunction", - "pygetset", - "pyimpl", - "pylib", - "pymath", - "pymember", - "PyMethod", - "PyModule", - "pyname", - "pyobj", - "PyObject", - "pypayload", - "PyProperty", - "pyref", - "PyResult", - "pyslot", - "PyStaticMethod", - "pystone", - "pystr", - "pystruct", - "pystructseq", - "pytrace", - "pytype", - "reducelib", - "richcompare", - "RustPython", "significand", - "struc", - "summands", // plural of summand - "sysmodule", - "tracebacks", - "typealiases", - "unhashable", - "uninit", + "summands", "unraisable", - "unresizable", - "varint", "wasi", - "zelf", + "weaked", // unix "posixshmem", "shm", "CLOEXEC", - "codeset", "endgrent", "gethrvtime", "getrusage", - "nanosleep", "sigaction", - "sighandler", "WRLCK", // win32 - "birthtime", - "IFEXEC", - // "stat" - "FIRMLINK", - // CPython internal names - "sysdict", - "settraceallthreads", - "setprofileallthreads" + "IFEXEC" ], // flagWords - list of words to be always considered incorrect "flagWords": [ diff --git a/.gitattributes b/.gitattributes index d1dd182a9b0..d076a34f977 100644 --- a/.gitattributes +++ b/.gitattributes @@ -4,5 +4,67 @@ Cargo.lock linguist-generated vm/src/stdlib/ast/gen.rs linguist-generated -merge Lib/*.py text working-tree-encoding=UTF-8 eol=LF **/*.rs text working-tree-encoding=UTF-8 eol=LF -*.pck binary crates/rustpython_doc_db/src/*.inc.rs linguist-generated=true + +# Binary data types +*.aif binary +*.aifc binary +*.aiff binary +*.au binary +*.bmp binary +*.exe binary +*.icns binary +*.gif binary +*.ico binary +*.jpg binary +*.pck binary +*.pdf binary +*.png binary +*.psd binary +*.tar binary +*.wav binary +*.whl binary +*.zip binary + +# Text files that should not be subject to eol conversion +[attr]noeol -text + +Lib/test/cjkencodings/* noeol +Lib/test/tokenizedata/coding20731.py noeol +Lib/test/decimaltestdata/*.decTest noeol +Lib/test/test_email/data/*.txt noeol +Lib/test/xmltestdata/* noeol + +# Shell scripts should have LF even on Windows because of Cygwin +Lib/venv/scripts/common/activate text eol=lf +Lib/venv/scripts/posix/* text eol=lf + +# CRLF files +[attr]dos text eol=crlf + +# Language aware diff headers +# https://tekin.co.uk/2020/10/better-git-diff-output-for-ruby-python-elixir-and-more +# https://gist.github.com/tekin/12500956bd56784728e490d8cef9cb81 +*.css diff=css +*.html diff=html +*.py diff=python +*.md diff=markdown + +# Generated files +# https://github.com/github/linguist/blob/master/docs/overrides.md +# +# To always hide generated files in local diffs, mark them as binary: +# $ git config diff.generated.binary true +# +[attr]generated linguist-generated=true diff=generated + +Lib/_opcode_metadata.py generated +Lib/keyword.py generated +Lib/idlelib/help.html generated +Lib/test/certdata/*.pem generated +Lib/test/certdata/*.0 generated +Lib/test/levenshtein_examples.json generated +Lib/test/test_stable_abi_ctypes.py generated +Lib/token.py generated + +.github/workflows/*.lock.yml linguist-generated=true merge=ours \ No newline at end of file diff --git a/.github/actions/install-linux-deps/action.yml b/.github/actions/install-linux-deps/action.yml new file mode 100644 index 00000000000..7900060fb29 --- /dev/null +++ b/.github/actions/install-linux-deps/action.yml @@ -0,0 +1,49 @@ +# This action installs a few dependencies necessary to build RustPython on Linux. +# It can be configured depending on which libraries are needed: +# +# ``` +# - uses: ./.github/actions/install-linux-deps +# with: +# gcc-multilib: true +# musl-tools: false +# ``` +# +# See the `inputs` section for all options and their defaults. Note that you must checkout the +# repository before you can use this action. +# +# This action will only install dependencies when the current operating system is Linux. It will do +# nothing on any other OS (macOS, Windows). + +name: Install Linux dependencies +description: Installs the dependencies necessary to build RustPython on Linux. +inputs: + gcc-multilib: + description: Install gcc-multilib (gcc-multilib) + required: false + default: "false" + musl-tools: + description: Install musl-tools (musl-tools) + required: false + default: "false" + gcc-aarch64-linux-gnu: + description: Install gcc-aarch64-linux-gnu (gcc-aarch64-linux-gnu) + required: false + default: "false" + clang: + description: Install clang (clang) + required: false + default: "false" +runs: + using: composite + steps: + - name: Install Linux dependencies + shell: bash + if: ${{ runner.os == 'Linux' }} + run: > + sudo apt-get update + + sudo apt-get install --no-install-recommends + ${{ fromJSON(inputs.gcc-multilib) && 'gcc-multilib' || '' }} + ${{ fromJSON(inputs.musl-tools) && 'musl-tools' || '' }} + ${{ fromJSON(inputs.clang) && 'clang' || '' }} + ${{ fromJSON(inputs.gcc-aarch64-linux-gnu) && 'gcc-aarch64-linux-gnu linux-libc-dev-arm64-cross libc6-dev-arm64-cross' || '' }} diff --git a/.github/actions/install-macos-deps/action.yml b/.github/actions/install-macos-deps/action.yml new file mode 100644 index 00000000000..46abef197a4 --- /dev/null +++ b/.github/actions/install-macos-deps/action.yml @@ -0,0 +1,47 @@ +# This action installs a few dependencies necessary to build RustPython on macOS. By default it installs +# autoconf, automake and libtool, but can be configured depending on which libraries are needed: +# +# ``` +# - uses: ./.github/actions/install-macos-deps +# with: +# openssl: true +# libtool: false +# ``` +# +# See the `inputs` section for all options and their defaults. Note that you must checkout the +# repository before you can use this action. +# +# This action will only install dependencies when the current operating system is macOS. It will do +# nothing on any other OS (Linux, Windows). + +name: Install macOS dependencies +description: Installs the dependencies necessary to build RustPython on macOS. +inputs: + autoconf: + description: Install autoconf (autoconf) + required: false + default: "true" + automake: + description: Install automake (automake) + required: false + default: "true" + libtool: + description: Install libtool (libtool) + required: false + default: "true" + openssl: + description: Install openssl (openssl@3) + required: false + default: "false" +runs: + using: composite + steps: + - name: Install macOS dependencies + shell: bash + if: ${{ runner.os == 'macOS' }} + run: > + brew install + ${{ fromJSON(inputs.autoconf) && 'autoconf' || '' }} + ${{ fromJSON(inputs.automake) && 'automake' || '' }} + ${{ fromJSON(inputs.libtool) && 'libtool' || '' }} + ${{ fromJSON(inputs.openssl) && 'openssl@3' || '' }} diff --git a/.github/aw/actions-lock.json b/.github/aw/actions-lock.json new file mode 100644 index 00000000000..ad986cbd051 --- /dev/null +++ b/.github/aw/actions-lock.json @@ -0,0 +1,14 @@ +{ + "entries": { + "actions/github-script@v8": { + "repo": "actions/github-script", + "version": "v8", + "sha": "ed597411d8f924073f98dfc5c65a23a2325f34cd" + }, + "github/gh-aw/actions/setup@v0.43.22": { + "repo": "github/gh-aw/actions/setup", + "version": "v0.43.22", + "sha": "fe858c3e14589bf396594a0b106e634d9065823e" + } + } +} diff --git a/.github/dependabot.yml b/.github/dependabot.yml index b3b7b446e4a..4995153001e 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,15 +1,167 @@ +# cspell:ignore manyhow tinyvec zeroize version: 2 updates: - package-ecosystem: cargo directory: / schedule: interval: weekly + cooldown: + default-days: 7 + semver-major-days: 30 + semver-minor-days: 7 + semver-patch-days: 3 + groups: + criterion: + patterns: + - "criterion*" + crypto: + patterns: + - "digest" + - "md-5" + - "sha-1" + - "sha2" + - "sha3" + - "blake2" + - "hmac" + - "pbkdf2" + futures: + patterns: + - "futures*" + get-size2: + patterns: + - "get-size*2" + iana-time-zone: + patterns: + - "iana-time-zone*" + jiff: + patterns: + - "jiff*" + lexical: + patterns: + - "lexical*" + libffi: + patterns: + - "libffi*" + malachite: + patterns: + - "malachite*" + manyhow: + patterns: + - "manyhow*" + num: + patterns: + - "num-bigint" + - "num-complex" + - "num-integer" + - "num-iter" + - "num-rational" + - "num-traits" + num_enum: + patterns: + - "num_enum*" + openssl: + patterns: + - "openssl*" + parking_lot: + patterns: + - "parking_lot*" + phf: + patterns: + - "phf*" + plotters: + patterns: + - "plotters*" + portable-atomic: + patterns: + - "portable-atomic*" + pyo3: + patterns: + - "pyo3*" + quote-use: + patterns: + - "quote-use*" + random: + patterns: + - "getrandom" + - "mt19937" + - "rand*" + rayon: + patterns: + - "rayon*" + regex: + patterns: + - "regex*" + result-like: + patterns: + - "result-like*" + security-framework: + patterns: + - "security-framework*" + serde: + patterns: + - "serde" + - "serde_core" + - "serde_derive" + system-configuration: + patterns: + - "system-configuration*" + thiserror: + patterns: + - "thiserror*" + time: + patterns: + - "time*" + tinyvec: + patterns: + - "tinyvec*" + tls_codec: + patterns: + - "tls_codec*" + toml: + patterns: + - "toml*" + wasm-bindgen: + patterns: + - "wasm-bindgen*" + wasmtime: + patterns: + - "cranelift*" + - "wasmtime*" + webpki-root: + patterns: + - "webpki-root*" + windows: + patterns: + - "windows*" + zerocopy: + patterns: + - "zerocopy*" + zeroize: + patterns: + - "zeroize*" ignore: # TODO: Remove when we use ruff from crates.io # for some reason dependabot only updates the Cargo.lock file when dealing # with git dependencies. i.e. not updating the version in Cargo.toml - - dependency-name: "ruff_*" + - dependency-name: "ruff_*" - package-ecosystem: github-actions directory: / schedule: interval: weekly + cooldown: + default-days: 7 + - package-ecosystem: npm + directory: / + schedule: + interval: weekly + cooldown: + default-days: 7 + semver-major-days: 30 + semver-minor-days: 7 + semver-patch-days: 3 + - package-ecosystem: pre-commit + directory: / + schedule: + interval: weekly + cooldown: + default-days: 7 diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 74db871139c..04983981966 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -8,119 +8,66 @@ on: name: CI +permissions: + contents: read + # Cancel previous workflows if they are the same workflow on same ref (branch/tags) # with the same event (push/pull_request) even they are in progress. # This setting will help reduce the number of duplicated workflows. concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} + group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.event.pull_request.number || github.sha }} cancel-in-progress: true env: - CARGO_ARGS: --no-default-features --features stdlib,importlib,stdio,encodings,sqlite,ssl-rustls - CARGO_ARGS_NO_SSL: --no-default-features --features stdlib,importlib,stdio,encodings,sqlite + CARGO_ARGS: --no-default-features --features stdlib,importlib,stdio,encodings,sqlite,ssl-rustls,host_env + CARGO_ARGS_NO_SSL: --no-default-features --features stdlib,importlib,stdio,encodings,sqlite,host_env # Crates excluded from workspace builds: # - rustpython_wasm: requires wasm target # - rustpython-compiler-source: deprecated # - rustpython-venvlauncher: Windows-only WORKSPACE_EXCLUDES: --exclude rustpython_wasm --exclude rustpython-compiler-source --exclude rustpython-venvlauncher - # Skip additional tests on Windows. They are checked on Linux and MacOS. - # test_glob: many failing tests - # test_pathlib: panic by surrogate chars - # test_posixpath: OSError: (22, 'The filename, directory name, or volume label syntax is incorrect. (os error 123)') - # test_venv: couple of failing tests - WINDOWS_SKIPS: >- - test_asyncio - test_glob - test_rlcompleter - test_pathlib - test_posixpath - test_venv - # PLATFORM_INDEPENDENT_TESTS are tests that do not depend on the underlying OS. They are currently - # only run on Linux to speed up the CI. - PLATFORM_INDEPENDENT_TESTS: >- - test__colorize - test_array - test_asyncgen - test_binop - test_bisect - test_bool - test_bytes - test_call - test_class - test_cmath - test_collections - test_complex - test_contains - test_copy - test_dataclasses - test_decimal - test_decorators - test_defaultdict - test_deque - test_dict - test_dictcomps - test_dictviews - test_dis - test_enumerate - test_exception_variations - test_float - test_format - test_fractions - test_genericalias - test_genericclass - test_grammar - test_range - test_index - test_int - test_int_literal - test_isinstance - test_iter - test_iterlen - test_itertools - test_json - test_keyword - test_keywordonlyarg - test_list - test_long - test_longexp - test_math - test_operator - test_ordered_dict - test_pow - test_raise - test_richcmp - test_scope - test_set - test_slice - test_sort - test_string - test_string_literals - test_strtod - test_structseq - test_subclassinit - test_super - test_syntax - test_tuple - test_types - test_unary - test_unpack - test_weakref - test_yield_from - ENV_POLLUTING_TESTS_COMMON: >- - ENV_POLLUTING_TESTS_LINUX: >- - test.test_multiprocessing_fork.test_processes - test.test_multiprocessing_forkserver.test_processes - test.test_multiprocessing_spawn.test_processes - ENV_POLLUTING_TESTS_MACOS: >- - test.test_multiprocessing_forkserver.test_processes - test.test_multiprocessing_spawn.test_processes - ENV_POLLUTING_TESTS_WINDOWS: >- - # Python version targeted by the CI. - PYTHON_VERSION: "3.14.2" X86_64_PC_WINDOWS_MSVC_OPENSSL_LIB_DIR: C:\Program Files\OpenSSL\lib\VC\x64\MD X86_64_PC_WINDOWS_MSVC_OPENSSL_INCLUDE_DIR: C:\Program Files\OpenSSL\include + CARGO_INCREMENTAL: 0 + CARGO_TERM_COLOR: always jobs: + determine_changes: + name: Determine changes + runs-on: ubuntu-latest + outputs: + # Flag that is raised when any rust code is changed. + rust_code: ${{ steps.check_rust_code.outputs.changed }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Determine merge base + id: merge_base + run: | + sha=$(git merge-base HEAD "origin/${BASE_REF}") + echo "sha=${sha}" >> "$GITHUB_OUTPUT" + env: + BASE_REF: ${{ github.event.pull_request.base.ref || 'main' }} + + - name: Check if there was any code related change + id: check_rust_code + run: | + if git diff --quiet "${MERGE_BASE}...HEAD" -- \ + ':!Lib/**' \ + ':!scripts/**' \ + ':!extra_tests/**' \ + ':.github/workflows/ci.yaml' \ + ; then + echo "changed=false" >> "$GITHUB_OUTPUT" + else + echo "changed=true" >> "$GITHUB_OUTPUT" + fi + env: + MERGE_BASE: ${{ steps.merge_base.outputs.sha }} + rust_tests: if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }} env: @@ -133,29 +80,46 @@ jobs: os: [macos-latest, ubuntu-latest, windows-2025] fail-fast: false steps: - - uses: actions/checkout@v6.0.2 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@stable with: components: clippy - - uses: Swatinem/rust-cache@v2 - - name: Set up the Mac environment - run: brew install autoconf automake libtool - if: runner.os == 'macOS' + - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + with: + save-if: ${{ github.ref == 'refs/heads/main' }} + + - name: Install macOS dependencies + uses: ./.github/actions/install-macos-deps - name: run clippy run: cargo clippy ${{ env.CARGO_ARGS }} --workspace --all-targets ${{ env.WORKSPACE_EXCLUDES }} -- -Dwarnings - name: run rust tests run: cargo test --workspace ${{ env.WORKSPACE_EXCLUDES }} --verbose --features threading ${{ env.CARGO_ARGS }} - if: runner.os != 'macOS' - - name: run rust tests - run: cargo test --workspace ${{ env.WORKSPACE_EXCLUDES }} --exclude rustpython-jit --verbose --features threading ${{ env.CARGO_ARGS }} - if: runner.os == 'macOS' - name: check compilation without threading run: cargo check ${{ env.CARGO_ARGS }} + - run: cargo doc --locked + if: runner.os == 'Linux' + + - name: check compilation without host_env (sandbox mode) + run: | + cargo check -p rustpython-vm --no-default-features --features compiler + cargo check -p rustpython-stdlib --no-default-features --features compiler + cargo build --no-default-features --features stdlib,importlib,stdio,encodings,freeze-stdlib + if: runner.os == 'Linux' + + - name: sandbox smoke test + run: | + target/debug/rustpython extra_tests/snippets/sandbox_smoke.py + target/debug/rustpython extra_tests/snippets/stdlib_re.py + if: runner.os == 'Linux' + - name: Test openssl build run: cargo build --no-default-features --features ssl-openssl if: runner.os == 'Linux' @@ -180,87 +144,81 @@ jobs: PYTHONPATH: scripts if: runner.os == 'Linux' - - name: prepare Intel MacOS build - uses: dtolnay/rust-toolchain@stable - with: + cargo_check: + name: cargo check + runs-on: ${{ matrix.os }} + needs: + - determine_changes + if: | + !contains(github.event.pull_request.labels.*.name, 'skip:ci') && + needs.determine_changes.outputs.rust_code == 'true' + strategy: + matrix: + include: + - os: ubuntu-latest + target: aarch64-linux-android + - os: ubuntu-latest + target: i686-unknown-linux-gnu + dependencies: + gcc-multilib: true + - os: ubuntu-latest + target: i686-unknown-linux-musl + dependencies: + musl-tools: true + - os: ubuntu-latest + target: wasm32-wasip2 + - os: ubuntu-latest + target: x86_64-unknown-freebsd + - os: ubuntu-latest + target: aarch64-unknown-linux-gnu + dependencies: + gcc-aarch64-linux-gnu: true + - os: macos-latest + target: aarch64-apple-ios + - os: macos-latest target: x86_64-apple-darwin - if: runner.os == 'macOS' - - name: Check compilation for Intel MacOS - run: cargo check --target x86_64-apple-darwin - if: runner.os == 'macOS' - - name: prepare iOS build - uses: dtolnay/rust-toolchain@stable + fail-fast: false + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - target: aarch64-apple-ios - if: runner.os == 'macOS' - - name: Check compilation for iOS - run: cargo check --target aarch64-apple-ios ${{ env.CARGO_ARGS_NO_SSL }} - if: runner.os == 'macOS' + persist-credentials: false - exotic_targets: - if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }} - name: Ensure compilation on various targets - runs-on: ubuntu-latest - timeout-minutes: 30 - steps: - - uses: actions/checkout@v6.0.2 - - uses: dtolnay/rust-toolchain@stable + - name: Install dependencies + uses: ./.github/actions/install-linux-deps + # zizmor has an issue with dynamic `with` + # with: ${{ matrix.dependencies || fromJSON('{}') }} with: - target: i686-unknown-linux-gnu + gcc-multilib: ${{ matrix.dependencies.gcc-multilib || false }} + musl-tools: ${{ matrix.dependencies.musl-tools || false }} + gcc-aarch64-linux-gnu: ${{ matrix.dependencies.gcc-aarch64-linux-gnu || false }} - - name: Install gcc-multilib and musl-tools - run: sudo apt-get update && sudo apt-get install gcc-multilib musl-tools - - name: Check compilation for x86 32bit - run: cargo check --target i686-unknown-linux-gnu ${{ env.CARGO_ARGS_NO_SSL }} + - name: Restore cache + uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 + if: ${{ github.ref != 'refs/heads/main' }} # Never restore on main + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + # key won't match, will rely on restore-keys + key: cargo-check-${{ runner.os }}-${{ matrix.target }} + restore-keys: | + cargo-check-${{ runner.os }}-${{ matrix.target }}- - uses: dtolnay/rust-toolchain@stable with: - target: aarch64-linux-android + target: ${{ matrix.target }} - name: Setup Android NDK + if: ${{ matrix.target == 'aarch64-linux-android' }} id: setup-ndk uses: nttld/setup-ndk@v1 with: ndk-version: r27 add-to-path: true - - name: Check compilation for android - run: cargo check --target aarch64-linux-android ${{ env.CARGO_ARGS_NO_SSL }} - env: - CC_aarch64_linux_android: ${{ steps.setup-ndk.outputs.ndk-path }}/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android24-clang - AR_aarch64_linux_android: ${{ steps.setup-ndk.outputs.ndk-path }}/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-ar - CARGO_TARGET_AARCH64_LINUX_ANDROID_LINKER: ${{ steps.setup-ndk.outputs.ndk-path }}/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android24-clang - - - uses: dtolnay/rust-toolchain@stable - with: - target: aarch64-unknown-linux-gnu - - - name: Install gcc-aarch64-linux-gnu - run: sudo apt install gcc-aarch64-linux-gnu - - name: Check compilation for aarch64 linux gnu - run: cargo check --target aarch64-unknown-linux-gnu ${{ env.CARGO_ARGS_NO_SSL }} - - - uses: dtolnay/rust-toolchain@stable - with: - target: i686-unknown-linux-musl - - - name: Check compilation for musl - run: cargo check --target i686-unknown-linux-musl ${{ env.CARGO_ARGS_NO_SSL }} - - - uses: dtolnay/rust-toolchain@stable - with: - target: x86_64-unknown-freebsd - - - name: Check compilation for freebsd - run: cargo check --target x86_64-unknown-freebsd ${{ env.CARGO_ARGS_NO_SSL }} - - - uses: dtolnay/rust-toolchain@stable - with: - target: x86_64-unknown-freebsd - - - name: Check compilation for freeBSD - run: cargo check --target x86_64-unknown-freebsd ${{ env.CARGO_ARGS_NO_SSL }} - # - name: Prepare repository for redox compilation # run: bash scripts/redox/uncomment-cargo.sh # - name: Check compilation for Redox @@ -269,149 +227,139 @@ jobs: # command: check # args: --ignore-rust-version + - name: Check compilation + run: cargo check --target "${{ matrix.target }}" ${{ env.CARGO_ARGS_NO_SSL }} + env: + CC_aarch64_linux_android: ${{ steps.setup-ndk.outputs.ndk-path }}/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android24-clang + AR_aarch64_linux_android: ${{ steps.setup-ndk.outputs.ndk-path }}/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-ar + CARGO_TARGET_AARCH64_LINUX_ANDROID_LINKER: ${{ steps.setup-ndk.outputs.ndk-path }}/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android24-clang + + - name: Save cache + if: ${{ github.ref == 'refs/heads/main' }} # only save on main + uses: actions/cache/save@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: cargo-check-${{ runner.os }}-${{ matrix.target }}-${{ hashFiles('**/Cargo.toml') }}-${{ hashFiles('Cargo.lock') }}-${{ github.sha }} + snippets_cpython: if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }} env: RUST_BACKTRACE: full + # Tests that can be flaky when running with multiple processes `-j 2`. We will use `-j 1` for these. + FLAKY_MP_TESTS: >- + test_class + test_concurrent_futures + test_eintr + test_multiprocessing_fork + test_multiprocessing_forkserver + test_multiprocessing_spawn name: Run snippets and cpython tests runs-on: ${{ matrix.os }} strategy: matrix: - os: [macos-latest, ubuntu-latest, windows-2025] + include: + - os: macos-latest + extra_test_args: + - '-u all' + env_polluting_tests: [] + skips: [] + timeout: 50 + - os: ubuntu-latest + extra_test_args: + - '-u all' + env_polluting_tests: [] + skips: [] + timeout: 60 + - os: windows-2025 + extra_test_args: [] # TODO: Enable '-u all' + env_polluting_tests: [] + skips: + - test_rlcompleter + - test_pathlib # panic by surrogate chars + - test_posixpath # OSError: (22, 'The filename, directory name, or volume label syntax is incorrect. (os error 123)') + - test_venv # couple of failing tests + timeout: 50 fail-fast: false steps: - - uses: actions/checkout@v6.0.2 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 - - uses: actions/setup-python@v6.2.0 + + - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 with: - python-version: ${{ env.PYTHON_VERSION }} - - name: Set up the Mac environment - run: brew install autoconf automake libtool openssl@3 - if: runner.os == 'macOS' - - name: build rustpython - run: cargo build --release --verbose --features=threading ${{ env.CARGO_ARGS }} - if: runner.os == 'macOS' - - name: build rustpython - run: cargo build --release --verbose --features=threading ${{ env.CARGO_ARGS }},jit - if: runner.os != 'macOS' - - uses: actions/setup-python@v6.2.0 + save-if: ${{ github.ref == 'refs/heads/main' }} + + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + + - name: Install macOS dependencies + uses: ./.github/actions/install-macos-deps with: - python-version: ${{ env.PYTHON_VERSION }} + openssl: true + + - name: build rustpython + run: cargo build --release --verbose --features=threading,jit ${{ env.CARGO_ARGS }} + - name: run snippets run: python -m pip install -r requirements.txt && pytest -v working-directory: ./extra_tests - - if: runner.os == 'Linux' - name: run cpython platform-independent tests - env: - RUSTPYTHON_SKIP_ENV_POLLUTERS: true - run: target/release/rustpython -m test -j 1 -u all --slowest --fail-env-changed --timeout 600 -v ${{ env.PLATFORM_INDEPENDENT_TESTS }} - timeout-minutes: 45 - - - if: runner.os == 'Linux' - name: run cpython platform-dependent tests (Linux) - env: - RUSTPYTHON_SKIP_ENV_POLLUTERS: true - run: target/release/rustpython -m test -j 1 -u all --slowest --fail-env-changed --timeout 600 -v -x ${{ env.PLATFORM_INDEPENDENT_TESTS }} - timeout-minutes: 45 + - name: Detect available cores + id: cores + shell: bash + run: | + cores=$(python -c 'print(__import__("os").process_cpu_count())') + echo "cores=${cores}" >> "$GITHUB_OUTPUT" - - if: runner.os == 'macOS' - name: run cpython platform-dependent tests (MacOS) + - name: Run CPython tests + run: | + target/release/rustpython -m test -j ${{ steps.cores.outputs.cores }} ${{ join(matrix.extra_test_args, ' ') }} --slowest --fail-env-changed --timeout 600 -v -x ${{ env.FLAKY_MP_TESTS }} ${{ join(matrix.skips, ' ') }} + timeout-minutes: ${{ matrix.timeout }} env: RUSTPYTHON_SKIP_ENV_POLLUTERS: true - run: target/release/rustpython -m test -j 1 --slowest --fail-env-changed --timeout 600 -v -x ${{ env.PLATFORM_INDEPENDENT_TESTS }} - timeout-minutes: 45 - - if: runner.os == 'Windows' - name: run cpython platform-dependent tests (windows partial - fixme) + - name: Run flaky MP CPython tests + run: | + target/release/rustpython -m test -j 1 ${{ join(matrix.extra_test_args, ' ') }} --slowest --fail-env-changed --timeout 600 -v ${{ env.FLAKY_MP_TESTS }} + timeout-minutes: ${{ matrix.timeout }} env: RUSTPYTHON_SKIP_ENV_POLLUTERS: true - run: target/release/rustpython -m test -j 1 --slowest --fail-env-changed --timeout 600 -v -x ${{ env.PLATFORM_INDEPENDENT_TESTS }} ${{ env.WINDOWS_SKIPS }} - timeout-minutes: 45 - - if: runner.os == 'Linux' - name: run cpython tests to check if env polluters have stopped polluting (Common/Linux) + - name: run cpython tests to check if env polluters have stopped polluting shell: bash run: | - for thing in ${{ env.ENV_POLLUTING_TESTS_COMMON }} ${{ env.ENV_POLLUTING_TESTS_LINUX }}; do - for i in $(seq 1 10); do - set +e - target/release/rustpython -m test -j 1 --slowest --fail-env-changed --timeout 600 -v ${thing} - exit_code=$? - set -e - if [ ${exit_code} -eq 3 ]; then - echo "Test ${thing} polluted the environment on attempt ${i}." - break - fi - done - if [ ${exit_code} -ne 3 ]; then - echo "Test ${thing} is no longer polluting the environment after ${i} attempts!" - echo "Please remove ${thing} from either ENV_POLLUTING_TESTS_COMMON or ENV_POLLUTING_TESTS_LINUX in '.github/workflows/ci.yaml'." - echo "Please also remove the skip decorators that include the word 'POLLUTERS' in ${thing}." - if [ ${exit_code} -ne 0 ]; then - echo "Test ${thing} failed with exit code ${exit_code}." - echo "Please investigate which test item in ${thing} is failing and either mark it as an expected failure or a skip." - fi - exit 1 - fi - done - timeout-minutes: 15 + IFS=' ' read -r -a target_array <<< "$TARGETS" - - if: runner.os == 'macOS' - name: run cpython tests to check if env polluters have stopped polluting (Common/macOS) - shell: bash - run: | - for thing in ${{ env.ENV_POLLUTING_TESTS_COMMON }} ${{ env.ENV_POLLUTING_TESTS_MACOS }}; do + for thing in "${target_array[@]}"; do for i in $(seq 1 10); do set +e - target/release/rustpython -m test -j 1 --slowest --fail-env-changed --timeout 600 -v ${thing} + target/release/rustpython -m test -j 1 --slowest --fail-env-changed --timeout 600 -v "${thing}" exit_code=$? set -e - if [ ${exit_code} -eq 3 ]; then + if [ "${exit_code}" -eq 3 ]; then echo "Test ${thing} polluted the environment on attempt ${i}." break fi done - if [ ${exit_code} -ne 3 ]; then + if [ "${exit_code}" -ne 3 ]; then echo "Test ${thing} is no longer polluting the environment after ${i} attempts!" - echo "Please remove ${thing} from either ENV_POLLUTING_TESTS_COMMON or ENV_POLLUTING_TESTS_MACOS in '.github/workflows/ci.yaml'." + echo "Please remove ${thing} from matrix.env_polluting_tests in '.github/workflows/ci.yaml'." echo "Please also remove the skip decorators that include the word 'POLLUTERS' in ${thing}." - if [ ${exit_code} -ne 0 ]; then - echo "Test ${thing} failed with exit code ${exit_code}." - echo "Please investigate which test item in ${thing} is failing and either mark it as an expected failure or a skip." - fi - exit 1 - fi - done - timeout-minutes: 15 - - - if: runner.os == 'Windows' - name: run cpython tests to check if env polluters have stopped polluting (Common/windows) - shell: bash - run: | - for thing in ${{ env.ENV_POLLUTING_TESTS_COMMON }} ${{ env.ENV_POLLUTING_TESTS_WINDOWS }}; do - for i in $(seq 1 10); do - set +e - target/release/rustpython -m test -j 1 --slowest --fail-env-changed --timeout 600 -v ${thing} - exit_code=$? - set -e - if [ ${exit_code} -eq 3 ]; then - echo "Test ${thing} polluted the environment on attempt ${i}." - break - fi - done - if [ ${exit_code} -ne 3 ]; then - echo "Test ${thing} is no longer polluting the environment after ${i} attempts!" - echo "Please remove ${thing} from either ENV_POLLUTING_TESTS_COMMON or ENV_POLLUTING_TESTS_WINDOWS in '.github/workflows/ci.yaml'." - echo "Please also remove the skip decorators that include the word 'POLLUTERS' in ${thing}." - if [ ${exit_code} -ne 0 ]; then + if [ "${exit_code}" -ne 0 ]; then echo "Test ${thing} failed with exit code ${exit_code}." echo "Please investigate which test item in ${thing} is failing and either mark it as an expected failure or a skip." fi exit 1 fi done + env: + TARGETS: ${{ join(matrix.env_polluting_tests, ' ') }} timeout-minutes: 15 - if: runner.os != 'Windows' @@ -420,70 +368,82 @@ jobs: mkdir site-packages target/release/rustpython --install-pip ensurepip --user target/release/rustpython -m pip install six + - name: Check that ensurepip succeeds. run: | target/release/rustpython -m ensurepip target/release/rustpython -c "import pip" + - if: runner.os != 'Windows' name: Check if pip inside venv is functional run: | target/release/rustpython -m venv testvenv testvenv/bin/rustpython -m pip install wheel - - if: runner.os != 'macOS' - name: Check whats_left is not broken - shell: bash - run: python -I scripts/whats_left.py --no-default-features --features "$(sed -e 's/--[^ ]*//g' <<< "${{ env.CARGO_ARGS }}" | tr -d '[:space:]'),threading,jit" - - if: runner.os == 'macOS' # TODO fix jit on macOS - name: Check whats_left is not broken (macOS) + + - name: Check whats_left is not broken shell: bash - run: python -I scripts/whats_left.py --no-default-features --features "$(sed -e 's/--[^ ]*//g' <<< "${{ env.CARGO_ARGS }}" | tr -d '[:space:]'),threading" # no jit on macOS for now + run: python -I scripts/whats_left.py ${{ env.CARGO_ARGS }} --features jit lint: - name: Check Rust code with clippy + name: Lint runs-on: ubuntu-latest + permissions: + contents: read + checks: write + pull-requests: write + security-events: write # for zizmor steps: - - uses: actions/checkout@v6.0.2 - - uses: dtolnay/rust-toolchain@stable + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - components: clippy + persist-credentials: false - - name: run clippy on wasm - run: cargo clippy --manifest-path=crates/wasm/Cargo.toml -- -Dwarnings + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + + - uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt - - name: Ensure docs generate no warnings - run: cargo doc --locked + - uses: cargo-bins/cargo-binstall@0b24824336e2b3800b0f89d9e08b2c08bfa3dcdd # v1.17.9 - - name: Ensure Lib/_opcode_metadata is updated + - name: cargo shear run: | - python scripts/generate_opcode_metadata.py - if [ -n "$(git status --porcelain)" ]; then - exit 1 - fi + cargo binstall --no-confirm cargo-shear + cargo shear - - name: Install ruff - uses: astral-sh/ruff-action@57714a7c8a2e59f32539362ba31877a1957dded1 # v3.5.1 - with: - version: "0.14.11" - args: "--version" + - name: actionlint + uses: reviewdog/action-actionlint@0d952c597ef8459f634d7145b0b044a9699e5e43 # v1.71.0 - - run: ruff check --diff + - name: zizmor + uses: zizmorcore/zizmor-action@71321a20a9ded102f6e9ce5718a2fcec2c4f70d8 # v0.5.2 - - run: ruff format --check + - name: restore prek cache + if: ${{ github.ref != 'refs/heads/main' }} # never restore on main + uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 + with: + key: prek-${{ hashFiles('.pre-commit-config.yaml') }} + path: ~/.cache/prek - - name: install prettier - run: yarn global add prettier && echo "$(yarn global bin)" >>$GITHUB_PATH + - name: prek + id: prek + uses: j178/prek-action@53276d8b0d10f8b6672aa85b4588c6921d0370cc # v2.0.1 + with: + cache: false + show-verbose-logs: false + continue-on-error: true - - name: check wasm code with prettier - # prettier doesn't handle ignore files very well: https://github.com/prettier/prettier/issues/8506 - run: cd wasm && git ls-files -z | xargs -0 prettier --check -u - # Keep cspell check as the last step. This is optional test. - - name: install extra dictionaries - run: npm install @cspell/dict-en_us @cspell/dict-cpp @cspell/dict-python @cspell/dict-rust @cspell/dict-win32 @cspell/dict-shell - - name: spell checker - uses: streetsidesoftware/cspell-action@v8 + - name: save prek cache + if: ${{ github.ref == 'refs/heads/main' }} # only save on main + uses: actions/cache/save@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: - files: "**/*.rs" - incremental_files_only: true + key: prek-${{ hashFiles('.pre-commit-config.yaml') }} + path: ~/.cache/prek + + - name: reviewdog + uses: reviewdog/action-suggester@aa38384ceb608d00f84b4690cacc83a5aba307ff # 1.24.0 + with: + level: warning + fail_level: error + cleanup: false miri: if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }} @@ -493,14 +453,18 @@ jobs: env: NIGHTLY_CHANNEL: nightly steps: - - uses: actions/checkout@v6.0.2 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - uses: dtolnay/rust-toolchain@master with: toolchain: ${{ env.NIGHTLY_CHANNEL }} components: miri - - uses: Swatinem/rust-cache@v2 + - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + with: + save-if: ${{ github.ref == 'refs/heads/main' }} - name: Run tests under miri run: cargo +${{ env.NIGHTLY_CHANNEL }} miri test -p rustpython-vm -- miri_test @@ -515,10 +479,21 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 steps: - - uses: actions/checkout@v6.0.2 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@stable + with: + components: clippy + + - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + with: + save-if: ${{ github.ref == 'refs/heads/main' }} + + - name: cargo clippy + run: cargo clippy --manifest-path=crates/wasm/Cargo.toml -- -Dwarnings - - uses: Swatinem/rust-cache@v2 - name: install wasm-pack run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh - name: install geckodriver @@ -526,18 +501,19 @@ jobs: wget https://github.com/mozilla/geckodriver/releases/download/v0.36.0/geckodriver-v0.36.0-linux64.tar.gz mkdir geckodriver tar -xzf geckodriver-v0.36.0-linux64.tar.gz -C geckodriver - - uses: actions/setup-python@v6.2.0 - with: - python-version: ${{ env.PYTHON_VERSION }} + + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - run: python -m pip install -r requirements.txt working-directory: ./wasm/tests - - uses: actions/setup-node@v6 + - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: cache: "npm" cache-dependency-path: "wasm/demo/package-lock.json" - name: run test run: | - export PATH=$PATH:`pwd`/../../geckodriver + driver_path="$(pwd)/../../geckodriver" + export PATH="$PATH:${driver_path}" npm install npm run test env: @@ -578,19 +554,29 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 steps: - - uses: actions/checkout@v6.0.2 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@stable with: target: wasm32-wasip1 - - uses: Swatinem/rust-cache@v2 + - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + with: + save-if: ${{ github.ref == 'refs/heads/main' }} + - name: Setup Wasmer uses: wasmerio/setup-wasmer@v3 + - name: Install clang - run: sudo apt-get update && sudo apt-get install clang -y + uses: ./.github/actions/install-linux-deps + with: + clang: true + - name: build rustpython run: cargo build --release --target wasm32-wasip1 --features freeze-stdlib,stdlib --verbose - name: run snippets - run: wasmer run --dir `pwd` target/wasm32-wasip1/release/rustpython.wasm -- `pwd`/extra_tests/snippets/stdlib_random.py + run: wasmer run --dir "$(pwd)" target/wasm32-wasip1/release/rustpython.wasm -- "$(pwd)/extra_tests/snippets/stdlib_random.py" - name: run cpython unittest - run: wasmer run --dir `pwd` target/wasm32-wasip1/release/rustpython.wasm -- `pwd`/Lib/test/test_int.py + run: wasmer run --dir "$(pwd)" target/wasm32-wasip1/release/rustpython.wasm -- "$(pwd)/Lib/test/test_int.py" diff --git a/.github/workflows/comment-commands.yml b/.github/workflows/comment-commands.yml index 3f3402270ea..eb9e2f96d1a 100644 --- a/.github/workflows/comment-commands.yml +++ b/.github/workflows/comment-commands.yml @@ -18,4 +18,6 @@ jobs: steps: # Using REST API and not `gh issue edit`. https://github.com/cli/cli/issues/6235#issuecomment-1243487651 - run: | - curl -H "Authorization: token ${{ github.token }}" -d '{"assignees": ["${{ github.event.comment.user.login }}"]}' https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/assignees + curl -H "Authorization: token ${{ github.token }}" -d '{"assignees": ["${{ env.USER }}"]}' https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/assignees + env: + USER: ${{ github.event.comment.user.login }} diff --git a/.github/workflows/cron-ci.yaml b/.github/workflows/cron-ci.yaml index 0a546595a8c..2e648e12cb7 100644 --- a/.github/workflows/cron-ci.yaml +++ b/.github/workflows/cron-ci.yaml @@ -12,8 +12,7 @@ on: name: Periodic checks/tasks env: - CARGO_ARGS: --no-default-features --features stdlib,importlib,encodings,ssl-rustls,jit - PYTHON_VERSION: "3.14.2" + CARGO_ARGS: --no-default-features --features stdlib,importlib,stdio,encodings,ssl-rustls,jit,host_env jobs: # codecov collects code coverage data from the rust tests, python snippets and python test suite. @@ -24,15 +23,18 @@ jobs: # Disable this scheduled job when running on a fork. if: ${{ github.repository == 'RustPython/RustPython' || github.event_name != 'schedule' }} steps: - - uses: actions/checkout@v6.0.2 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@stable - uses: taiki-e/install-action@cargo-llvm-cov - - uses: actions/setup-python@v6.2.0 - with: - python-version: ${{ env.PYTHON_VERSION }} + + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - run: sudo apt-get update && sudo apt-get -y install lcov - name: Run cargo-llvm-cov with Rust tests. - run: cargo llvm-cov --no-report --workspace --exclude rustpython_wasm --exclude rustpython-compiler-source --exclude rustpython-venvlauncher --verbose --no-default-features --features stdlib,importlib,encodings,ssl-rustls,jit + run: cargo llvm-cov --no-report --workspace --exclude rustpython_wasm --exclude rustpython-compiler-source --exclude rustpython-venvlauncher --verbose --no-default-features --features stdlib,importlib,stdio,encodings,ssl-rustls,jit,host_env - name: Run cargo-llvm-cov with Python snippets. run: python scripts/cargo-llvm-cov.py continue-on-error: true @@ -45,7 +47,7 @@ jobs: if: ${{ github.event_name != 'pull_request' }} uses: codecov/codecov-action@v5 with: - file: ./codecov.lcov + files: ./codecov.lcov testdata: name: Collect regression test data @@ -53,7 +55,10 @@ jobs: # Disable this scheduled job when running on a fork. if: ${{ github.repository == 'RustPython/RustPython' || github.event_name != 'schedule' }} steps: - - uses: actions/checkout@v6.0.2 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: true + - uses: dtolnay/rust-toolchain@stable - name: build rustpython run: cargo build --release --verbose @@ -85,11 +90,14 @@ jobs: # Disable this scheduled job when running on a fork. if: ${{ github.repository == 'RustPython/RustPython' || github.event_name != 'schedule' }} steps: - - uses: actions/checkout@v6.0.2 - - uses: dtolnay/rust-toolchain@stable - - uses: actions/setup-python@v6.2.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - python-version: ${{ env.PYTHON_VERSION }} + persist-credentials: true + + - uses: dtolnay/rust-toolchain@stable + + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - name: build rustpython run: cargo build --release --verbose - name: Collect what is left data @@ -143,11 +151,14 @@ jobs: # Disable this scheduled job when running on a fork. if: ${{ github.repository == 'RustPython/RustPython' || github.event_name != 'schedule' }} steps: - - uses: actions/checkout@v6.0.2 - - uses: dtolnay/rust-toolchain@stable - - uses: actions/setup-python@v6.2.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - python-version: ${{ env.PYTHON_VERSION }} + persist-credentials: true + + - uses: dtolnay/rust-toolchain@stable + + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - run: cargo install cargo-criterion - name: build benchmarks run: cargo build --release --benches @@ -158,12 +169,12 @@ jobs: - name: restructure generated files run: | cd ./target/criterion/reports - find -type d -name cpython | xargs rm -rf - find -type d -name rustpython | xargs rm -rf - find -mindepth 2 -maxdepth 2 -name violin.svg | xargs rm -rf - find -type f -not -name violin.svg | xargs rm -rf - for file in $(find -type f -name violin.svg); do mv $file $(echo $file | sed -E "s_\./([^/]+)/([^/]+)/violin\.svg_./\1/\2.svg_"); done - find -mindepth 2 -maxdepth 2 -type d | xargs rm -rf + find . -type d -name cpython -print0 | xargs -0 rm -rf + find . -type d -name rustpython -print0 | xargs -0 rm -rf + find . -mindepth 2 -maxdepth 2 -name violin.svg -print0 | xargs -0 rm -rf + find . -type f -not -name violin.svg -print0 | xargs -0 rm -rf + find . -type f -name violin.svg -exec sh -c 'for file; do mv "$file" "$(echo "$file" | sed -E "s_\./([^/]+)/([^/]+)/violin\.svg_./\1/\2.svg_")"; done' _ {} + + find . -mindepth 2 -maxdepth 2 -type d -print0 | xargs -0 rm -rf cd .. mv reports/* . rmdir reports diff --git a/.github/workflows/lib-deps-check.yaml b/.github/workflows/lib-deps-check.yaml index a4b7128d830..b009c427df7 100644 --- a/.github/workflows/lib-deps-check.yaml +++ b/.github/workflows/lib-deps-check.yaml @@ -4,10 +4,10 @@ on: pull_request_target: types: [opened, synchronize, reopened] paths: - - 'Lib/**' + - "Lib/**" concurrency: - group: lib-deps-${{ github.event.pull_request.number }} + group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.event.pull_request.number }} cancel-in-progress: true jobs: @@ -18,11 +18,12 @@ jobs: timeout-minutes: 10 steps: - name: Checkout base branch - uses: actions/checkout@v6.0.2 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: # Use base branch for scripts (security: don't run PR code with elevated permissions) ref: ${{ github.event.pull_request.base.ref }} fetch-depth: 0 + persist-credentials: false - name: Fetch PR head run: | @@ -33,9 +34,20 @@ jobs: # Checkout only Lib/ directory from PR head for accurate comparison git checkout ${{ github.event.pull_request.head.sha }} -- Lib/ - - name: Checkout CPython + - name: Get target CPython version + id: cpython-version run: | - git clone --depth 1 --branch v3.14.2 https://github.com/python/cpython.git cpython + version=$(cat .python-version) + echo "version=${version}" >> "$GITHUB_OUTPUT" + + - name: Checkout CPython + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + repository: python/cpython + path: cpython + ref: "v${{ steps.cpython-version.outputs.version }}" + fetch-depth: 1 + persist-credentials: false - name: Get changed Lib files id: changed-files @@ -70,36 +82,26 @@ jobs: - name: Setup Python if: steps.changed-files.outputs.modules != '' - uses: actions/setup-python@v6.2.0 - with: - python-version: "3.12" + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 - name: Run deps check if: steps.changed-files.outputs.modules != '' id: deps-check run: | # Run deps for all modules at once - python scripts/update_lib deps ${{ steps.changed-files.outputs.modules }} --depth 2 > /tmp/deps_output.txt 2>&1 || true - - # Read output for GitHub Actions - echo "deps_output<> $GITHUB_OUTPUT - cat /tmp/deps_output.txt >> $GITHUB_OUTPUT - echo "EOF" >> $GITHUB_OUTPUT - - # Check if there's any meaningful output - if [ -s /tmp/deps_output.txt ]; then - echo "has_output=true" >> $GITHUB_OUTPUT - else - echo "has_output=false" >> $GITHUB_OUTPUT - fi + echo "deps_output<> "$GITHUB_OUTPUT" + output=$(python scripts/update_lib deps "${MODULES}" --depth 2 2>&1 || true) + echo "$output" >> "$GITHUB_OUTPUT" + echo "EOF" >> "$GITHUB_OUTPUT" + env: + MODULES: ${{ steps.changed-files.outputs.modules }} - name: Post comment - if: steps.deps-check.outputs.has_output == 'true' - uses: marocchino/sticky-pull-request-comment@v2 + if: steps.deps-check.outputs.deps_output != '' + uses: marocchino/sticky-pull-request-comment@v3 with: header: lib-deps-check number: ${{ github.event.pull_request.number }} - recreate: true message: | ## 📦 Library Dependencies @@ -113,7 +115,7 @@ jobs: - name: Remove comment if no Lib changes if: steps.changed-files.outputs.modules == '' - uses: marocchino/sticky-pull-request-comment@v2 + uses: marocchino/sticky-pull-request-comment@v3 with: header: lib-deps-check number: ${{ github.event.pull_request.number }} diff --git a/.github/workflows/pr-auto-commit.yaml b/.github/workflows/pr-auto-commit.yaml deleted file mode 100644 index f34dd724e4c..00000000000 --- a/.github/workflows/pr-auto-commit.yaml +++ /dev/null @@ -1,120 +0,0 @@ -name: Auto-format PR - -# This workflow triggers when a PR is opened/updated -on: - pull_request_target: - types: [opened, synchronize, reopened] - branches: - - main - - release - -concurrency: - group: auto-format-${{ github.event.pull_request.number }} - cancel-in-progress: true - -jobs: - auto_format: - permissions: - contents: write - pull-requests: write - runs-on: ubuntu-latest - timeout-minutes: 60 - steps: - - name: Checkout PR branch - uses: actions/checkout@v6.0.2 - with: - ref: ${{ github.event.pull_request.head.sha }} - repository: ${{ github.event.pull_request.head.repo.full_name }} - token: ${{ secrets.AUTO_COMMIT_PAT }} - fetch-depth: 0 - - - name: Setup Rust - uses: dtolnay/rust-toolchain@stable - with: - components: rustfmt - - - name: Configure git - run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - echo "" > /tmp/committed_commands.txt - - - name: Run cargo fmt - run: | - echo "Running cargo fmt --all on PR #${{ github.event.pull_request.number }}" - cargo fmt --all - if [ -n "$(git status --porcelain)" ]; then - git add -u - git commit -m "Auto-format: cargo fmt --all" - echo "- \`cargo fmt --all\`" >> /tmp/committed_commands.txt - fi - - - name: Install ruff - uses: astral-sh/ruff-action@57714a7c8a2e59f32539362ba31877a1957dded1 # v3.5.1 - with: - version: "0.14.11" - args: "--version" - - - name: Run ruff format - run: | - ruff format - if [ -n "$(git status --porcelain)" ]; then - git add -u - git commit -m "Auto-format: ruff format" - echo "- \`ruff format\`" >> /tmp/committed_commands.txt - fi - - - name: Run ruff check import sorting - run: | - ruff check --select I --fix - if [ -n "$(git status --porcelain)" ]; then - git add -u - git commit -m "Auto-format: ruff check --select I --fix" - echo "- \`ruff check --select I --fix\`" >> /tmp/committed_commands.txt - fi - - - name: Run generate_opcode_metadata.py - run: | - python scripts/generate_opcode_metadata.py - if [ -n "$(git status --porcelain)" ]; then - git add -u - git commit -m "Auto-generate: generate_opcode_metadata.py" - echo "- \`python scripts/generate_opcode_metadata.py\`" >> /tmp/committed_commands.txt - fi - - - name: Check for changes - id: check-changes - run: | - if [ "$(git rev-parse HEAD)" != "${{ github.event.pull_request.head.sha }}" ]; then - echo "has_changes=true" >> $GITHUB_OUTPUT - else - echo "has_changes=false" >> $GITHUB_OUTPUT - fi - - - name: Push formatting changes - if: steps.check-changes.outputs.has_changes == 'true' - run: | - git push origin HEAD:${{ github.event.pull_request.head.ref }} - - - name: Read committed commands - id: committed-commands - if: steps.check-changes.outputs.has_changes == 'true' - run: | - echo "list<> $GITHUB_OUTPUT - cat /tmp/committed_commands.txt >> $GITHUB_OUTPUT - echo "EOF" >> $GITHUB_OUTPUT - - - name: Comment on PR - if: steps.check-changes.outputs.has_changes == 'true' - uses: marocchino/sticky-pull-request-comment@v2 - with: - number: ${{ github.event.pull_request.number }} - message: | - **Code has been automatically formatted** - - The code in this PR has been formatted using: - ${{ steps.committed-commands.outputs.list }} - Please pull the latest changes before pushing again: - ```bash - git pull origin ${{ github.event.pull_request.head.ref }} - ``` diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9a6d0ad9838..ae6c9da9b74 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,83 +12,83 @@ on: required: false default: true -permissions: - contents: write - env: - CARGO_ARGS: --no-default-features --features stdlib,importlib,encodings,sqlite,ssl X86_64_PC_WINDOWS_MSVC_OPENSSL_LIB_DIR: C:\Program Files\OpenSSL\lib\VC\x64\MD X86_64_PC_WINDOWS_MSVC_OPENSSL_INCLUDE_DIR: C:\Program Files\OpenSSL\include jobs: build: - runs-on: ${{ matrix.platform.runner }} + runs-on: ${{ matrix.os }} # Disable this scheduled job when running on a fork. if: ${{ github.repository == 'RustPython/RustPython' || github.event_name != 'schedule' }} strategy: matrix: - platform: - - runner: ubuntu-latest + include: + - os: ubuntu-latest target: x86_64-unknown-linux-gnu -# - runner: ubuntu-latest -# target: i686-unknown-linux-gnu -# - runner: ubuntu-latest -# target: aarch64-unknown-linux-gnu -# - runner: ubuntu-latest -# target: armv7-unknown-linux-gnueabi -# - runner: ubuntu-latest -# target: s390x-unknown-linux-gnu -# - runner: ubuntu-latest -# target: powerpc64le-unknown-linux-gnu - - runner: macos-latest + - os: macos-latest target: aarch64-apple-darwin -# - runner: macos-latest -# target: x86_64-apple-darwin - - runner: windows-2025 + - os: windows-2025 target: x86_64-pc-windows-msvc -# - runner: windows-2025 -# target: i686-pc-windows-msvc -# - runner: windows-2025 -# target: aarch64-pc-windows-msvc + # - os: ubuntu-latest + # target: i686-unknown-linux-gnu + # - os: ubuntu-latest + # target: aarch64-unknown-linux-gnu + # - os: ubuntu-latest + # target: armv7-unknown-linux-gnueabi + # - os: ubuntu-latest + # target: s390x-unknown-linux-gnu + # - os: ubuntu-latest + # target: powerpc64le-unknown-linux-gnu + # - os: macos-latest + # target: x86_64-apple-darwin + # - os: windows-2025 + # target: i686-pc-windows-msvc + # - os: windows-2025 + # target: aarch64-pc-windows-msvc fail-fast: false steps: - - uses: actions/checkout@v6.0.2 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@stable - - uses: cargo-bins/cargo-binstall@main + with: + target: ${{ matrix.target }} - - name: Set up Environment - shell: bash - run: rustup target add ${{ matrix.platform.target }} - - name: Set up MacOS Environment - run: brew install autoconf automake libtool - if: runner.os == 'macOS' + - name: Install macOS dependencies + uses: ./.github/actions/install-macos-deps + with: + autoconf: true + automake: true + libtool: true - name: Build RustPython - run: cargo build --release --target=${{ matrix.platform.target }} --verbose --features=threading ${{ env.CARGO_ARGS }} - if: runner.os == 'macOS' - - name: Build RustPython - run: cargo build --release --target=${{ matrix.platform.target }} --verbose --features=threading ${{ env.CARGO_ARGS }},jit - if: runner.os != 'macOS' + run: cargo build --release --target=${{ matrix.target }} --verbose --no-default-features --features stdlib,stdio,importlib,encodings,sqlite,host_env,ssl-rustls,threading,jit - name: Rename Binary - run: cp target/${{ matrix.platform.target }}/release/rustpython target/rustpython-release-${{ runner.os }}-${{ matrix.platform.target }} + run: cp target/${{ matrix.target }}/release/rustpython target/rustpython-release-${{ runner.os }}-${{ matrix.target }} if: runner.os != 'Windows' + - name: Rename Binary - run: cp target/${{ matrix.platform.target }}/release/rustpython.exe target/rustpython-release-${{ runner.os }}-${{ matrix.platform.target }}.exe + run: cp target/${{ matrix.target }}/release/rustpython.exe target/rustpython-release-${{ runner.os }}-${{ matrix.target }}.exe if: runner.os == 'Windows' - name: Upload Binary Artifacts - uses: actions/upload-artifact@v6.0.0 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: - name: rustpython-release-${{ runner.os }}-${{ matrix.platform.target }} - path: target/rustpython-release-${{ runner.os }}-${{ matrix.platform.target }}* + name: rustpython-release-${{ runner.os }}-${{ matrix.target }} + path: target/rustpython-release-${{ runner.os }}-${{ matrix.target }}* build-wasm: runs-on: ubuntu-latest # Disable this scheduled job when running on a fork. if: ${{ github.repository == 'RustPython/RustPython' || github.event_name != 'schedule' }} steps: - - uses: actions/checkout@v6.0.2 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@stable with: targets: wasm32-wasip1 @@ -100,16 +100,22 @@ jobs: run: cp target/wasm32-wasip1/release/rustpython.wasm target/rustpython-release-wasm32-wasip1.wasm - name: Upload Binary Artifacts - uses: actions/upload-artifact@v6.0.0 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: rustpython-release-wasm32-wasip1 path: target/rustpython-release-wasm32-wasip1.wasm - name: install wasm-pack run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh - - uses: actions/setup-node@v6 - - uses: mwilliamson/setup-wabt-action@v3 - with: { wabt-version: "1.0.30" } + + - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 + with: + package-manager-cache: false + + - uses: mwilliamson/setup-wabt-action@febe2a12b7ccb999a6e5d953a8362a3b7ffcf148 # v3.2.0 + with: + wabt-version: "1.0.30" + - name: build demo run: | npm install @@ -117,6 +123,7 @@ jobs: env: NODE_OPTIONS: "--openssl-legacy-provider" working-directory: ./wasm/demo + - name: build notebook demo run: | npm install @@ -125,8 +132,10 @@ jobs: env: NODE_OPTIONS: "--openssl-legacy-provider" working-directory: ./wasm/notebook + - name: Deploy demo to Github Pages - uses: peaceiris/actions-gh-pages@v4 + if: ${{ github.repository == 'RustPython/RustPython' }} + uses: peaceiris/actions-gh-pages@4f9cc6602d3f66b9c108549d475ec49e8ef4d45e # v4.0.0 with: deploy_key: ${{ secrets.ACTIONS_DEMO_DEPLOY_KEY }} publish_dir: ./wasm/demo/dist @@ -139,30 +148,28 @@ jobs: if: ${{ github.repository == 'RustPython/RustPython' || github.event_name != 'schedule' }} needs: [build, build-wasm] steps: - - uses: actions/checkout@v6.0.2 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - name: Download Binary Artifacts - uses: actions/download-artifact@v7.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: path: bin pattern: rustpython-* merge-multiple: true - name: Create Lib Archive - run: | - zip -r bin/rustpython-lib.zip Lib/ + run: zip -r bin/rustpython-lib.zip Lib/ - name: List Binaries run: | ls -lah bin/ file bin/* + - name: Create Release - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - tag: ${{ github.ref_name }} - run: ${{ github.run_number }} run: | - if [[ "${{ github.event.inputs.pre-release }}" == "false" ]]; then + if [[ "${PRE_RELEASE_INPUT}" == "false" ]]; then RELEASE_TYPE_NAME=Release PRERELEASE_ARG= else @@ -179,3 +186,8 @@ jobs: --generate-notes \ $PRERELEASE_ARG \ bin/rustpython-release-* + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + tag: ${{ github.ref_name }} + run: ${{ github.run_number }} + PRE_RELEASE_INPUT: ${{ github.event.inputs.pre-release }} diff --git a/.github/workflows/update-doc-db.yml b/.github/workflows/update-doc-db.yml index 1fd3b930985..bcb766e0a35 100644 --- a/.github/workflows/update-doc-db.yml +++ b/.github/workflows/update-doc-db.yml @@ -1,7 +1,6 @@ name: Update doc DB -permissions: - contents: write +permissions: {} on: workflow_dispatch: @@ -9,11 +8,11 @@ on: python-version: description: Target python version to generate doc db for type: string - default: "3.14.2" - ref: - description: Branch to commit to (leave empty for current branch) + default: "3.14.3" + base-ref: + description: Base branch to create the update branch from type: string - default: "" + default: "main" defaults: run: @@ -21,6 +20,8 @@ defaults: jobs: generate: + permissions: + contents: read runs-on: ${{ matrix.os }} strategy: matrix: @@ -29,7 +30,7 @@ jobs: - windows-latest - macos-latest steps: - - uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98 # v6.0.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false sparse-checkout: | @@ -42,7 +43,7 @@ jobs: - name: Generate docs run: python crates/doc/generate.py - - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: doc-db-${{ inputs.python-version }}-${{ matrix.os }} path: "crates/doc/generated/*.json" @@ -53,20 +54,30 @@ jobs: merge: runs-on: ubuntu-latest needs: generate + permissions: + contents: write + pull-requests: write steps: - - uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98 # v6.0.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - ref: ${{ inputs.ref || github.ref }} - token: ${{ secrets.AUTO_COMMIT_PAT }} + persist-credentials: true + ref: ${{ inputs.base-ref }} + + - name: Create update branch + run: git switch -c "update-doc-${PYTHON_VERSION}" + env: + PYTHON_VERSION: ${{ inputs.python-version }} - name: Download generated doc DBs - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: pattern: "doc-db-${{ inputs.python-version }}-**" path: crates/doc/generated/ merge-multiple: true - name: Transform JSON + env: + PYTHON_VERSION: ${{ inputs.python-version }} run: | # Merge all artifacts jq -s "add" --sort-keys crates/doc/generated/*.json > crates/doc/generated/merged.json @@ -79,7 +90,7 @@ jobs: echo -n '' > $OUTPUT_FILE echo '// This file was auto-generated by `.github/workflows/update-doc-db.yml`.' >> $OUTPUT_FILE - echo "// CPython version: ${{ inputs.python-version }}" >> $OUTPUT_FILE + echo "// CPython version: ${PYTHON_VERSION}" >> $OUTPUT_FILE echo '// spell-checker: disable' >> $OUTPUT_FILE echo '' >> $OUTPUT_FILE @@ -88,7 +99,7 @@ jobs: cat crates/doc/generated/raw_entries.txt >> $OUTPUT_FILE echo '};' >> $OUTPUT_FILE - - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: doc-db-${{ inputs.python-version }} path: "crates/doc/src/data.inc.rs" @@ -96,13 +107,20 @@ jobs: retention-days: 7 overwrite: true - - name: Commit and push (non-main branches only) - if: github.ref != 'refs/heads/main' && inputs.ref != 'main' + - name: Commit, push and create PR + env: + GH_TOKEN: ${{ github.token }} + PYTHON_VERSION: ${{ inputs.python-version }} + BASE_REF: ${{ inputs.base-ref }} run: | git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" if [ -n "$(git status --porcelain)" ]; then git add crates/doc/src/data.inc.rs - git commit -m "Update doc DB for CPython ${{ inputs.python-version }}" - git push + git commit -m "Update doc DB for CPython ${PYTHON_VERSION}" + git push -u origin HEAD + gh pr create \ + --base "${BASE_REF}" \ + --title "Update doc DB for CPython ${PYTHON_VERSION}" \ + --body "Auto-generated by update-doc-db workflow." fi diff --git a/.github/workflows/update-libs-status.yaml b/.github/workflows/update-libs-status.yaml new file mode 100644 index 00000000000..db32c7a3e94 --- /dev/null +++ b/.github/workflows/update-libs-status.yaml @@ -0,0 +1,96 @@ +name: Updated libs status + +on: + push: + branches: + - main + paths: + - "Lib/**" + workflow_dispatch: + +permissions: + contents: read + issues: write + +env: + ISSUE_ID: "6839" + +jobs: + update-issue: + runs-on: ubuntu-latest + if: ${{ github.repository == 'RustPython/RustPython' }} + steps: + - name: Clone RustPython + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + path: rustpython + persist-credentials: false + sparse-checkout: |- + Lib + scripts/update_lib + .python-version + + - name: Get target CPython version + id: cpython-version + run: | + version=$(cat rustpython/.python-version) + echo "version=${version}" >> "$GITHUB_OUTPUT" + + - name: Clone CPython + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + repository: python/cpython + path: cpython + ref: "v${{ steps.cpython-version.outputs.version }}" + persist-credentials: false + sparse-checkout: | + Lib + + - name: Get current date + id: current_date + run: | + now=$(date -u +"%Y-%m-%d %H:%M:%S") + echo "date=$now" >> "$GITHUB_OUTPUT" + + - name: Write body prefix + run: | + cat > body.txt < + + ## Summary + + Check \`scripts/update_lib\` for tools. As a note, the current latest Python version is \`${{ steps.cpython-version.outputs.version }}\`. + + Previous versions' issues as reference + - 3.13: #5529 + + + + ## Details + + ${{ steps.current_date.outputs.date }} (UTC) + \`\`\`shell + $ python3 scripts/update_lib todo --done + \`\`\` + EOF + + - name: Run todo + run: python3 rustpython/scripts/update_lib todo --cpython cpython --lib rustpython/Lib --done >> body.txt + + - name: Update GH issue + run: gh issue edit ${{ env.ISSUE_ID }} --body-file ../body.txt + env: + GH_TOKEN: ${{ github.token }} + working-directory: rustpython + + diff --git a/.github/workflows/upgrade-pylib.lock.yml b/.github/workflows/upgrade-pylib.lock.yml new file mode 100644 index 00000000000..1f4e42341e2 --- /dev/null +++ b/.github/workflows/upgrade-pylib.lock.yml @@ -0,0 +1,1093 @@ +# +# ___ _ _ +# / _ \ | | (_) +# | |_| | __ _ ___ _ __ | |_ _ ___ +# | _ |/ _` |/ _ \ '_ \| __| |/ __| +# | | | | (_| | __/ | | | |_| | (__ +# \_| |_/\__, |\___|_| |_|\__|_|\___| +# __/ | +# _ _ |___/ +# | | | | / _| | +# | | | | ___ _ __ _ __| |_| | _____ ____ +# | |/\| |/ _ \ '__| |/ /| _| |/ _ \ \ /\ / / ___| +# \ /\ / (_) | | | | ( | | | | (_) \ V V /\__ \ +# \/ \/ \___/|_| |_|\_\|_| |_|\___/ \_/\_/ |___/ +# +# This file was automatically generated by gh-aw (v0.43.22). DO NOT EDIT. +# +# To update this file, edit the corresponding .md file and run: +# gh aw compile +# Not all edits will cause changes to this file. +# +# For more information: https://github.github.com/gh-aw/introduction/overview/ +# +# Pick an out-of-sync Python library from the todo list and upgrade it +# by running `scripts/update_lib quick`, then open a pull request. +# +# frontmatter-hash: 3129480d6628afe028911bb8b31b6bb3b5eb251e395f00ed0677922cd21727cb + +name: "Upgrade Python Library" +"on": + workflow_dispatch: + inputs: + name: + description: Module name to upgrade (leave empty to auto-pick) + required: false + type: string + +permissions: {} + +concurrency: + group: "gh-aw-${{ github.workflow }}" + +run-name: "Upgrade Python Library" + +env: + ISSUE_ID: "6839" + PYTHON_VERSION: v3.14.3 + +# Cache configuration from frontmatter was processed and added to the main job steps + +jobs: + activation: + runs-on: ubuntu-slim + permissions: + contents: read + outputs: + comment_id: "" + comment_repo: "" + steps: + - name: Setup Scripts + uses: github/gh-aw/actions/setup@48d8fdfddc8cad854ac0c70ceb573f09fb8f9c9b # v0.62.5 + with: + destination: /opt/gh-aw/actions + - name: Check workflow file timestamps + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_WORKFLOW_FILE: "upgrade-pylib.lock.yml" + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/check_workflow_timestamp_api.cjs'); + await main(); + + agent: + needs: activation + runs-on: ubuntu-latest + permissions: + contents: read + issues: read + pull-requests: read + concurrency: + group: "gh-aw-copilot-${{ github.workflow }}" + env: + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + GH_AW_ASSETS_ALLOWED_EXTS: "" + GH_AW_ASSETS_BRANCH: "" + GH_AW_ASSETS_MAX_SIZE_KB: 0 + GH_AW_MCP_LOG_DIR: /tmp/gh-aw/mcp-logs/safeoutputs + GH_AW_SAFE_OUTPUTS: /opt/gh-aw/safeoutputs/outputs.jsonl + GH_AW_SAFE_OUTPUTS_CONFIG_PATH: /opt/gh-aw/safeoutputs/config.json + GH_AW_SAFE_OUTPUTS_TOOLS_PATH: /opt/gh-aw/safeoutputs/tools.json + outputs: + checkout_pr_success: ${{ steps.checkout-pr.outputs.checkout_pr_success || 'true' }} + has_patch: ${{ steps.collect_output.outputs.has_patch }} + model: ${{ steps.generate_aw_info.outputs.model }} + output: ${{ steps.collect_output.outputs.output }} + output_types: ${{ steps.collect_output.outputs.output_types }} + secret_verification_result: ${{ steps.validate-secret.outputs.verification_result }} + steps: + - name: Setup Scripts + uses: github/gh-aw/actions/setup@48d8fdfddc8cad854ac0c70ceb573f09fb8f9c9b # v0.62.5 + with: + destination: /opt/gh-aw/actions + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: Setup Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.14' + - name: Create gh-aw temp directory + run: bash /opt/gh-aw/actions/create_gh_aw_tmp_dir.sh + # Cache configuration from frontmatter processed below + - name: Cache (cpython-lib-${{ env.PYTHON_VERSION }}) + uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 + with: + key: cpython-lib-${{ env.PYTHON_VERSION }} + path: cpython + restore-keys: | + cpython-lib- + - name: Configure Git credentials + env: + REPO_NAME: ${{ github.repository }} + SERVER_URL: ${{ github.server_url }} + run: | + git config --global user.email "github-actions[bot]@users.noreply.github.com" + git config --global user.name "github-actions[bot]" + # Re-authenticate git with GitHub token + SERVER_URL_STRIPPED="${SERVER_URL#https://}" + git remote set-url origin "https://x-access-token:${{ github.token }}@${SERVER_URL_STRIPPED}/${REPO_NAME}.git" + echo "Git configured with standard GitHub Actions identity" + - name: Checkout PR branch + id: checkout-pr + if: | + github.event.pull_request + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + with: + github-token: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/checkout_pr_branch.cjs'); + await main(); + - name: Generate agentic run info + id: generate_aw_info + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + with: + script: | + const fs = require('fs'); + + const awInfo = { + engine_id: "copilot", + engine_name: "GitHub Copilot CLI", + model: process.env.GH_AW_MODEL_AGENT_COPILOT || "", + version: "", + agent_version: "0.0.409", + cli_version: "v0.43.22", + workflow_name: "Upgrade Python Library", + experimental: false, + supports_tools_allowlist: true, + supports_http_transport: true, + run_id: context.runId, + run_number: context.runNumber, + run_attempt: process.env.GITHUB_RUN_ATTEMPT, + repository: context.repo.owner + '/' + context.repo.repo, + ref: context.ref, + sha: context.sha, + actor: context.actor, + event_name: context.eventName, + staged: false, + allowed_domains: ["defaults","rust","python"], + firewall_enabled: true, + awf_version: "v0.16.4", + awmg_version: "", + steps: { + firewall: "squid" + }, + created_at: new Date().toISOString() + }; + + // Write to /tmp/gh-aw directory to avoid inclusion in PR + const tmpPath = '/tmp/gh-aw/aw_info.json'; + fs.writeFileSync(tmpPath, JSON.stringify(awInfo, null, 2)); + console.log('Generated aw_info.json at:', tmpPath); + console.log(JSON.stringify(awInfo, null, 2)); + + // Set model as output for reuse in other steps/jobs + core.setOutput('model', awInfo.model); + - name: Validate COPILOT_GITHUB_TOKEN secret + id: validate-secret + run: /opt/gh-aw/actions/validate_multi_secret.sh COPILOT_GITHUB_TOKEN 'GitHub Copilot CLI' https://github.github.com/gh-aw/reference/engines/#github-copilot-default + env: + COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + - name: Install GitHub Copilot CLI + run: /opt/gh-aw/actions/install_copilot_cli.sh 0.0.409 + - name: Install awf binary + run: bash /opt/gh-aw/actions/install_awf_binary.sh v0.16.4 + - name: Determine automatic lockdown mode for GitHub MCP server + id: determine-automatic-lockdown + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_GITHUB_TOKEN: ${{ secrets.GH_AW_GITHUB_TOKEN }} + GH_AW_GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN }} + with: + script: | + const determineAutomaticLockdown = require('/opt/gh-aw/actions/determine_automatic_lockdown.cjs'); + await determineAutomaticLockdown(github, context, core); + - name: Download container images + run: bash /opt/gh-aw/actions/download_docker_images.sh ghcr.io/github/gh-aw-firewall/agent:0.16.4 ghcr.io/github/gh-aw-firewall/squid:0.16.4 ghcr.io/github/gh-aw-mcpg:v0.1.4 ghcr.io/github/github-mcp-server:v0.30.3 node:lts-alpine + - name: Write Safe Outputs Config + run: | + mkdir -p /opt/gh-aw/safeoutputs + mkdir -p /tmp/gh-aw/safeoutputs + mkdir -p /tmp/gh-aw/mcp-logs/safeoutputs + cat > /opt/gh-aw/safeoutputs/config.json << 'GH_AW_SAFE_OUTPUTS_CONFIG_EOF' + {"create_pull_request":{"expires":30},"missing_data":{},"missing_tool":{},"noop":{"max":1}} + GH_AW_SAFE_OUTPUTS_CONFIG_EOF + cat > /opt/gh-aw/safeoutputs/tools.json << 'GH_AW_SAFE_OUTPUTS_TOOLS_EOF' + [ + { + "description": "Create a new GitHub pull request to propose code changes. Use this after making file edits to submit them for review and merging. The PR will be created from the current branch with your committed changes. For code review comments on an existing PR, use create_pull_request_review_comment instead. CONSTRAINTS: Maximum 1 pull request(s) can be created. Title will be prefixed with \"Update \". Labels [pylib-sync] will be automatically added.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "body": { + "description": "Detailed PR description in Markdown. Include what changes were made, why, testing notes, and any breaking changes. Do NOT repeat the title as a heading.", + "type": "string" + }, + "branch": { + "description": "Source branch name containing the changes. If omitted, uses the current working branch.", + "type": "string" + }, + "labels": { + "description": "Labels to categorize the PR (e.g., 'enhancement', 'bugfix'). Labels must exist in the repository.", + "items": { + "type": "string" + }, + "type": "array" + }, + "title": { + "description": "Concise PR title describing the changes. Follow repository conventions (e.g., conventional commits). The title appears as the main heading.", + "type": "string" + } + }, + "required": [ + "title", + "body" + ], + "type": "object" + }, + "name": "create_pull_request" + }, + { + "description": "Report that a tool or capability needed to complete the task is not available, or share any information you deem important about missing functionality or limitations. Use this when you cannot accomplish what was requested because the required functionality is missing or access is restricted.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "alternatives": { + "description": "Any workarounds, manual steps, or alternative approaches the user could take (max 256 characters).", + "type": "string" + }, + "reason": { + "description": "Explanation of why this tool is needed or what information you want to share about the limitation (max 256 characters).", + "type": "string" + }, + "tool": { + "description": "Optional: Name or description of the missing tool or capability (max 128 characters). Be specific about what functionality is needed.", + "type": "string" + } + }, + "required": [ + "reason" + ], + "type": "object" + }, + "name": "missing_tool" + }, + { + "description": "Log a transparency message when no significant actions are needed. Use this to confirm workflow completion and provide visibility when analysis is complete but no changes or outputs are required (e.g., 'No issues found', 'All checks passed'). This ensures the workflow produces human-visible output even when no other actions are taken.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "message": { + "description": "Status or completion message to log. Should explain what was analyzed and the outcome (e.g., 'Code review complete - no issues found', 'Analysis complete - all tests passing').", + "type": "string" + } + }, + "required": [ + "message" + ], + "type": "object" + }, + "name": "noop" + }, + { + "description": "Report that data or information needed to complete the task is not available. Use this when you cannot accomplish what was requested because required data, context, or information is missing.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "alternatives": { + "description": "Any workarounds, manual steps, or alternative approaches the user could take (max 256 characters).", + "type": "string" + }, + "context": { + "description": "Additional context about the missing data or where it should come from (max 256 characters).", + "type": "string" + }, + "data_type": { + "description": "Type or description of the missing data or information (max 128 characters). Be specific about what data is needed.", + "type": "string" + }, + "reason": { + "description": "Explanation of why this data is needed to complete the task (max 256 characters).", + "type": "string" + } + }, + "required": [], + "type": "object" + }, + "name": "missing_data" + } + ] + GH_AW_SAFE_OUTPUTS_TOOLS_EOF + cat > /opt/gh-aw/safeoutputs/validation.json << 'GH_AW_SAFE_OUTPUTS_VALIDATION_EOF' + { + "create_pull_request": { + "defaultMax": 1, + "fields": { + "body": { + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 65000 + }, + "branch": { + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 256 + }, + "labels": { + "type": "array", + "itemType": "string", + "itemSanitize": true, + "itemMaxLength": 128 + }, + "title": { + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 128 + } + } + }, + "missing_tool": { + "defaultMax": 20, + "fields": { + "alternatives": { + "type": "string", + "sanitize": true, + "maxLength": 512 + }, + "reason": { + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 256 + }, + "tool": { + "type": "string", + "sanitize": true, + "maxLength": 128 + } + } + }, + "noop": { + "defaultMax": 1, + "fields": { + "message": { + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 65000 + } + } + } + } + GH_AW_SAFE_OUTPUTS_VALIDATION_EOF + - name: Generate Safe Outputs MCP Server Config + id: safe-outputs-config + run: | + # Generate a secure random API key (360 bits of entropy, 40+ chars) + # Mask immediately to prevent timing vulnerabilities + API_KEY=$(openssl rand -base64 45 | tr -d '/+=') + echo "::add-mask::${API_KEY}" + + PORT=3001 + + # Set outputs for next steps + { + echo "safe_outputs_api_key=${API_KEY}" + echo "safe_outputs_port=${PORT}" + } >> "$GITHUB_OUTPUT" + + echo "Safe Outputs MCP server will run on port ${PORT}" + + - name: Start Safe Outputs MCP HTTP Server + id: safe-outputs-start + env: + DEBUG: '*' + GH_AW_SAFE_OUTPUTS_PORT: ${{ steps.safe-outputs-config.outputs.safe_outputs_port }} + GH_AW_SAFE_OUTPUTS_API_KEY: ${{ steps.safe-outputs-config.outputs.safe_outputs_api_key }} + GH_AW_SAFE_OUTPUTS_TOOLS_PATH: /opt/gh-aw/safeoutputs/tools.json + GH_AW_SAFE_OUTPUTS_CONFIG_PATH: /opt/gh-aw/safeoutputs/config.json + GH_AW_MCP_LOG_DIR: /tmp/gh-aw/mcp-logs/safeoutputs + run: | + # Environment variables are set above to prevent template injection + export DEBUG + export GH_AW_SAFE_OUTPUTS_PORT + export GH_AW_SAFE_OUTPUTS_API_KEY + export GH_AW_SAFE_OUTPUTS_TOOLS_PATH + export GH_AW_SAFE_OUTPUTS_CONFIG_PATH + export GH_AW_MCP_LOG_DIR + + bash /opt/gh-aw/actions/start_safe_outputs_server.sh + + - name: Start MCP gateway + id: start-mcp-gateway + env: + GH_AW_SAFE_OUTPUTS: ${{ env.GH_AW_SAFE_OUTPUTS }} + GH_AW_SAFE_OUTPUTS_API_KEY: ${{ steps.safe-outputs-start.outputs.api_key }} + GH_AW_SAFE_OUTPUTS_PORT: ${{ steps.safe-outputs-start.outputs.port }} + GITHUB_MCP_LOCKDOWN: ${{ steps.determine-automatic-lockdown.outputs.lockdown == 'true' && '1' || '0' }} + GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + run: | + set -eo pipefail + mkdir -p /tmp/gh-aw/mcp-config + + # Export gateway environment variables for MCP config and gateway script + export MCP_GATEWAY_PORT="80" + export MCP_GATEWAY_DOMAIN="host.docker.internal" + MCP_GATEWAY_API_KEY=$(openssl rand -base64 45 | tr -d '/+=') + echo "::add-mask::${MCP_GATEWAY_API_KEY}" + export MCP_GATEWAY_API_KEY + export MCP_GATEWAY_PAYLOAD_DIR="/tmp/gh-aw/mcp-payloads" + mkdir -p "${MCP_GATEWAY_PAYLOAD_DIR}" + export DEBUG="*" + + export GH_AW_ENGINE="copilot" + export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network host -v /var/run/docker.sock:/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_API_KEY -e MCP_GATEWAY_PAYLOAD_DIR -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_LOCKDOWN -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e GH_AW_SAFE_OUTPUTS_PORT -e GH_AW_SAFE_OUTPUTS_API_KEY -v /tmp/gh-aw/mcp-payloads:/tmp/gh-aw/mcp-payloads:rw -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw ghcr.io/github/gh-aw-mcpg:v0.1.4' + + mkdir -p /home/runner/.copilot + cat << GH_AW_MCP_CONFIG_EOF | bash /opt/gh-aw/actions/start_mcp_gateway.sh + { + "mcpServers": { + "github": { + "type": "stdio", + "container": "ghcr.io/github/github-mcp-server:v0.30.3", + "env": { + "GITHUB_LOCKDOWN_MODE": "$GITHUB_MCP_LOCKDOWN", + "GITHUB_PERSONAL_ACCESS_TOKEN": "\${GITHUB_MCP_SERVER_TOKEN}", + "GITHUB_READ_ONLY": "1", + "GITHUB_TOOLSETS": "repos,issues,pull_requests" + } + }, + "safeoutputs": { + "type": "http", + "url": "http://host.docker.internal:$GH_AW_SAFE_OUTPUTS_PORT", + "headers": { + "Authorization": "\${GH_AW_SAFE_OUTPUTS_API_KEY}" + } + } + }, + "gateway": { + "port": $MCP_GATEWAY_PORT, + "domain": "${MCP_GATEWAY_DOMAIN}", + "apiKey": "${MCP_GATEWAY_API_KEY}", + "payloadDir": "${MCP_GATEWAY_PAYLOAD_DIR}" + } + } + GH_AW_MCP_CONFIG_EOF + - name: Generate workflow overview + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + with: + script: | + const { generateWorkflowOverview } = require('/opt/gh-aw/actions/generate_workflow_overview.cjs'); + await generateWorkflowOverview(core); + - name: Create prompt with built-in context + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_SAFE_OUTPUTS: ${{ env.GH_AW_SAFE_OUTPUTS }} + GH_AW_ENV_ISSUE_ID: ${{ env.ISSUE_ID }} + GH_AW_GITHUB_ACTOR: ${{ github.actor }} + GH_AW_GITHUB_EVENT_COMMENT_ID: ${{ github.event.comment.id }} + GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER: ${{ github.event.discussion.number }} + GH_AW_GITHUB_EVENT_INPUTS_NAME: ${{ github.event.inputs.name }} + GH_AW_GITHUB_EVENT_ISSUE_NUMBER: ${{ github.event.issue.number }} + GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }} + GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} + GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} + GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} + run: | + bash /opt/gh-aw/actions/create_prompt_first.sh + cat << 'GH_AW_PROMPT_EOF' > "$GH_AW_PROMPT" + + GH_AW_PROMPT_EOF + cat "/opt/gh-aw/prompts/xpia.md" >> "$GH_AW_PROMPT" + cat "/opt/gh-aw/prompts/temp_folder_prompt.md" >> "$GH_AW_PROMPT" + cat "/opt/gh-aw/prompts/markdown.md" >> "$GH_AW_PROMPT" + cat << 'GH_AW_PROMPT_EOF' >> "$GH_AW_PROMPT" + + GitHub API Access Instructions + + The gh CLI is NOT authenticated. Do NOT use gh commands for GitHub operations. + + + To create or modify GitHub resources (issues, discussions, pull requests, etc.), you MUST call the appropriate safe output tool. Simply writing content will NOT work - the workflow requires actual tool calls. + + Temporary IDs: Some safe output tools support a temporary ID field (usually named temporary_id) so you can reference newly-created items elsewhere in the SAME agent output (for example, using #aw_abc1 in a later body). + + **IMPORTANT - temporary_id format rules:** + - If you DON'T need to reference the item later, OMIT the temporary_id field entirely (it will be auto-generated if needed) + - If you DO need cross-references/chaining, you MUST match this EXACT validation regex: /^aw_[A-Za-z0-9]{3,8}$/i + - Format: aw_ prefix followed by 3 to 8 alphanumeric characters (A-Z, a-z, 0-9, case-insensitive) + - Valid alphanumeric characters: ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789 + - INVALID examples: aw_ab (too short), aw_123456789 (too long), aw_test-id (contains hyphen), aw_id_123 (contains underscore) + - VALID examples: aw_abc, aw_abc1, aw_Test123, aw_A1B2C3D4, aw_12345678 + - To generate valid IDs: use 3-8 random alphanumeric characters or omit the field to let the system auto-generate + + Do NOT invent other aw_* formats — downstream steps will reject them with validation errors matching against /^aw_[A-Za-z0-9]{3,8}$/i. + + Discover available tools from the safeoutputs MCP server. + + **Critical**: Tool calls write structured data that downstream jobs process. Without tool calls, follow-up actions will be skipped. + + **Note**: If you made no other safe output tool calls during this workflow execution, call the "noop" tool to provide a status message indicating completion or that no actions were needed. + + + + The following GitHub context information is available for this workflow: + {{#if __GH_AW_GITHUB_ACTOR__ }} + - **actor**: __GH_AW_GITHUB_ACTOR__ + {{/if}} + {{#if __GH_AW_GITHUB_REPOSITORY__ }} + - **repository**: __GH_AW_GITHUB_REPOSITORY__ + {{/if}} + {{#if __GH_AW_GITHUB_WORKSPACE__ }} + - **workspace**: __GH_AW_GITHUB_WORKSPACE__ + {{/if}} + {{#if __GH_AW_GITHUB_EVENT_ISSUE_NUMBER__ }} + - **issue-number**: #__GH_AW_GITHUB_EVENT_ISSUE_NUMBER__ + {{/if}} + {{#if __GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER__ }} + - **discussion-number**: #__GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER__ + {{/if}} + {{#if __GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER__ }} + - **pull-request-number**: #__GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER__ + {{/if}} + {{#if __GH_AW_GITHUB_EVENT_COMMENT_ID__ }} + - **comment-id**: __GH_AW_GITHUB_EVENT_COMMENT_ID__ + {{/if}} + {{#if __GH_AW_GITHUB_RUN_ID__ }} + - **workflow-run-id**: __GH_AW_GITHUB_RUN_ID__ + {{/if}} + + + GH_AW_PROMPT_EOF + cat << 'GH_AW_PROMPT_EOF' >> "$GH_AW_PROMPT" + + GH_AW_PROMPT_EOF + cat << 'GH_AW_PROMPT_EOF' >> "$GH_AW_PROMPT" + {{#runtime-import .github/workflows/upgrade-pylib.md}} + GH_AW_PROMPT_EOF + - name: Substitute placeholders + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_ENV_ISSUE_ID: ${{ env.ISSUE_ID }} + GH_AW_GITHUB_ACTOR: ${{ github.actor }} + GH_AW_GITHUB_EVENT_COMMENT_ID: ${{ github.event.comment.id }} + GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER: ${{ github.event.discussion.number }} + GH_AW_GITHUB_EVENT_INPUTS_NAME: ${{ github.event.inputs.name }} + GH_AW_GITHUB_EVENT_ISSUE_NUMBER: ${{ github.event.issue.number }} + GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }} + GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} + GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} + GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} + with: + script: | + const substitutePlaceholders = require('/opt/gh-aw/actions/substitute_placeholders.cjs'); + + // Call the substitution function + return await substitutePlaceholders({ + file: process.env.GH_AW_PROMPT, + substitutions: { + GH_AW_ENV_ISSUE_ID: process.env.GH_AW_ENV_ISSUE_ID, + GH_AW_GITHUB_ACTOR: process.env.GH_AW_GITHUB_ACTOR, + GH_AW_GITHUB_EVENT_COMMENT_ID: process.env.GH_AW_GITHUB_EVENT_COMMENT_ID, + GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER: process.env.GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER, + GH_AW_GITHUB_EVENT_INPUTS_NAME: process.env.GH_AW_GITHUB_EVENT_INPUTS_NAME, + GH_AW_GITHUB_EVENT_ISSUE_NUMBER: process.env.GH_AW_GITHUB_EVENT_ISSUE_NUMBER, + GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER: process.env.GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER, + GH_AW_GITHUB_REPOSITORY: process.env.GH_AW_GITHUB_REPOSITORY, + GH_AW_GITHUB_RUN_ID: process.env.GH_AW_GITHUB_RUN_ID, + GH_AW_GITHUB_WORKSPACE: process.env.GH_AW_GITHUB_WORKSPACE + } + }); + - name: Interpolate variables and render templates + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_ENV_ISSUE_ID: ${{ env.ISSUE_ID }} + GH_AW_GITHUB_EVENT_INPUTS_NAME: ${{ github.event.inputs.name }} + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/interpolate_prompt.cjs'); + await main(); + - name: Validate prompt placeholders + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + run: bash /opt/gh-aw/actions/validate_prompt_placeholders.sh + - name: Print prompt + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + run: bash /opt/gh-aw/actions/print_prompt_summary.sh + - name: Clean git credentials + run: bash /opt/gh-aw/actions/clean_git_credentials.sh + - name: Execute GitHub Copilot CLI + id: agentic_execution + # Copilot CLI tool arguments (sorted): + timeout-minutes: 45 + run: | + set -o pipefail + sudo -E awf --env-all --container-workdir "${GITHUB_WORKSPACE}" --allow-domains '*.pythonhosted.org,anaconda.org,api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,binstar.org,bootstrap.pypa.io,conda.anaconda.org,conda.binstar.org,crates.io,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,files.pythonhosted.org,github.com,host.docker.internal,index.crates.io,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,pip.pypa.io,ppa.launchpad.net,pypi.org,pypi.python.org,raw.githubusercontent.com,registry.npmjs.org,repo.anaconda.com,repo.continuum.io,s.symcb.com,s.symcd.com,security.ubuntu.com,sh.rustup.rs,static.crates.io,static.rust-lang.org,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com' --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --enable-host-access --image-tag 0.16.4 --skip-pull \ + -- '/usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --add-dir "${GITHUB_WORKSPACE}" --disable-builtin-mcps --allow-all-tools --allow-all-paths --share /tmp/gh-aw/sandbox/agent/logs/conversation.md --prompt "$(cat /tmp/gh-aw/aw-prompts/prompt.txt)"${GH_AW_MODEL_AGENT_COPILOT:+ --model "$GH_AW_MODEL_AGENT_COPILOT"}' \ + 2>&1 | tee /tmp/gh-aw/agent-stdio.log + env: + COPILOT_AGENT_RUNNER_TYPE: STANDALONE + COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + GH_AW_MCP_CONFIG: /home/runner/.copilot/mcp-config.json + GH_AW_MODEL_AGENT_COPILOT: ${{ vars.GH_AW_MODEL_AGENT_COPILOT || '' }} + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_SAFE_OUTPUTS: ${{ env.GH_AW_SAFE_OUTPUTS }} + GITHUB_HEAD_REF: ${{ github.head_ref }} + GITHUB_REF_NAME: ${{ github.ref_name }} + GITHUB_STEP_SUMMARY: ${{ env.GITHUB_STEP_SUMMARY }} + GITHUB_WORKSPACE: ${{ github.workspace }} + XDG_CONFIG_HOME: /home/runner + - name: Configure Git credentials + env: + REPO_NAME: ${{ github.repository }} + SERVER_URL: ${{ github.server_url }} + run: | + git config --global user.email "github-actions[bot]@users.noreply.github.com" + git config --global user.name "github-actions[bot]" + # Re-authenticate git with GitHub token + SERVER_URL_STRIPPED="${SERVER_URL#https://}" + git remote set-url origin "https://x-access-token:${{ github.token }}@${SERVER_URL_STRIPPED}/${REPO_NAME}.git" + echo "Git configured with standard GitHub Actions identity" + - name: Copy Copilot session state files to logs + if: always() + continue-on-error: true + run: | + # Copy Copilot session state files to logs folder for artifact collection + # This ensures they are in /tmp/gh-aw/ where secret redaction can scan them + SESSION_STATE_DIR="$HOME/.copilot/session-state" + LOGS_DIR="/tmp/gh-aw/sandbox/agent/logs" + if [ -d "$SESSION_STATE_DIR" ]; then + echo "Copying Copilot session state files from $SESSION_STATE_DIR to $LOGS_DIR" + mkdir -p "$LOGS_DIR" + cp -v "$SESSION_STATE_DIR"/*.jsonl "$LOGS_DIR/" 2>/dev/null || true + echo "Session state files copied successfully" + else + echo "No session-state directory found at $SESSION_STATE_DIR" + fi + - name: Stop MCP gateway + if: always() + continue-on-error: true + env: + MCP_GATEWAY_PORT: ${{ steps.start-mcp-gateway.outputs.gateway-port }} + MCP_GATEWAY_API_KEY: ${{ steps.start-mcp-gateway.outputs.gateway-api-key }} + GATEWAY_PID: ${{ steps.start-mcp-gateway.outputs.gateway-pid }} + run: | + bash /opt/gh-aw/actions/stop_mcp_gateway.sh "$GATEWAY_PID" + - name: Redact secrets in logs + if: always() + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/redact_secrets.cjs'); + await main(); + env: + GH_AW_SECRET_NAMES: 'COPILOT_GITHUB_TOKEN,GH_AW_GITHUB_MCP_SERVER_TOKEN,GH_AW_GITHUB_TOKEN,GITHUB_TOKEN' + SECRET_COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + SECRET_GH_AW_GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN }} + SECRET_GH_AW_GITHUB_TOKEN: ${{ secrets.GH_AW_GITHUB_TOKEN }} + SECRET_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Upload Safe Outputs + if: always() + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: safe-output + path: ${{ env.GH_AW_SAFE_OUTPUTS }} + if-no-files-found: warn + - name: Ingest agent output + id: collect_output + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_SAFE_OUTPUTS: ${{ env.GH_AW_SAFE_OUTPUTS }} + GH_AW_ALLOWED_DOMAINS: "*.pythonhosted.org,anaconda.org,api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,binstar.org,bootstrap.pypa.io,conda.anaconda.org,conda.binstar.org,crates.io,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,files.pythonhosted.org,github.com,host.docker.internal,index.crates.io,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,pip.pypa.io,ppa.launchpad.net,pypi.org,pypi.python.org,raw.githubusercontent.com,registry.npmjs.org,repo.anaconda.com,repo.continuum.io,s.symcb.com,s.symcd.com,security.ubuntu.com,sh.rustup.rs,static.crates.io,static.rust-lang.org,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com" + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_API_URL: ${{ github.api_url }} + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/collect_ndjson_output.cjs'); + await main(); + - name: Upload sanitized agent output + if: always() && env.GH_AW_AGENT_OUTPUT + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: agent-output + path: ${{ env.GH_AW_AGENT_OUTPUT }} + if-no-files-found: warn + - name: Upload engine output files + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: agent_outputs + path: | + /tmp/gh-aw/sandbox/agent/logs/ + /tmp/gh-aw/redacted-urls.log + if-no-files-found: ignore + - name: Parse agent logs for step summary + if: always() + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_AGENT_OUTPUT: /tmp/gh-aw/sandbox/agent/logs/ + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/parse_copilot_log.cjs'); + await main(); + - name: Parse MCP gateway logs for step summary + if: always() + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/parse_mcp_gateway_log.cjs'); + await main(); + - name: Print firewall logs + if: always() + continue-on-error: true + env: + AWF_LOGS_DIR: /tmp/gh-aw/sandbox/firewall/logs + run: | + # Fix permissions on firewall logs so they can be uploaded as artifacts + # AWF runs with sudo, creating files owned by root + sudo chmod -R a+r /tmp/gh-aw/sandbox/firewall/logs 2>/dev/null || true + awf logs summary | tee -a "$GITHUB_STEP_SUMMARY" + - name: Upload agent artifacts + if: always() + continue-on-error: true + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: agent-artifacts + path: | + /tmp/gh-aw/aw-prompts/prompt.txt + /tmp/gh-aw/aw_info.json + /tmp/gh-aw/mcp-logs/ + /tmp/gh-aw/sandbox/firewall/logs/ + /tmp/gh-aw/agent-stdio.log + /tmp/gh-aw/agent/ + /tmp/gh-aw/aw*.patch + if-no-files-found: ignore + + conclusion: + needs: + - activation + - agent + - detection + - safe_outputs + if: (always()) && (needs.agent.result != 'skipped') + runs-on: ubuntu-slim + permissions: + contents: read + discussions: write + issues: write + pull-requests: write + outputs: + noop_message: ${{ steps.noop.outputs.noop_message }} + tools_reported: ${{ steps.missing_tool.outputs.tools_reported }} + total_count: ${{ steps.missing_tool.outputs.total_count }} + steps: + - name: Setup Scripts + uses: github/gh-aw/actions/setup@48d8fdfddc8cad854ac0c70ceb573f09fb8f9c9b # v0.62.5 + with: + destination: /opt/gh-aw/actions + - name: Download agent output artifact + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: agent-output + path: /tmp/gh-aw/safeoutputs/ + - name: Setup agent output environment variable + run: | + mkdir -p /tmp/gh-aw/safeoutputs/ + find "/tmp/gh-aw/safeoutputs/" -type f -print + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/safeoutputs/agent_output.json" >> "$GITHUB_ENV" + - name: Process No-Op Messages + id: noop + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} + GH_AW_NOOP_MAX: 1 + GH_AW_WORKFLOW_NAME: "Upgrade Python Library" + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/noop.cjs'); + await main(); + - name: Record Missing Tool + id: missing_tool + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} + GH_AW_WORKFLOW_NAME: "Upgrade Python Library" + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/missing_tool.cjs'); + await main(); + - name: Handle Agent Failure + id: handle_agent_failure + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} + GH_AW_WORKFLOW_NAME: "Upgrade Python Library" + GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }} + GH_AW_WORKFLOW_ID: "upgrade-pylib" + GH_AW_SECRET_VERIFICATION_RESULT: ${{ needs.agent.outputs.secret_verification_result }} + GH_AW_CHECKOUT_PR_SUCCESS: ${{ needs.agent.outputs.checkout_pr_success }} + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/handle_agent_failure.cjs'); + await main(); + - name: Handle No-Op Message + id: handle_noop_message + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} + GH_AW_WORKFLOW_NAME: "Upgrade Python Library" + GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }} + GH_AW_NOOP_MESSAGE: ${{ steps.noop.outputs.noop_message }} + GH_AW_NOOP_REPORT_AS_ISSUE: "true" + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/handle_noop_message.cjs'); + await main(); + - name: Handle Create Pull Request Error + id: handle_create_pr_error + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} + GH_AW_WORKFLOW_NAME: "Upgrade Python Library" + GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/handle_create_pr_error.cjs'); + await main(); + - name: Update reaction comment with completion status + id: conclusion + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} + GH_AW_COMMENT_ID: ${{ needs.activation.outputs.comment_id }} + GH_AW_COMMENT_REPO: ${{ needs.activation.outputs.comment_repo }} + GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GH_AW_WORKFLOW_NAME: "Upgrade Python Library" + GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }} + GH_AW_DETECTION_CONCLUSION: ${{ needs.detection.result }} + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/notify_comment_error.cjs'); + await main(); + + detection: + needs: agent + if: needs.agent.outputs.output_types != '' || needs.agent.outputs.has_patch == 'true' + runs-on: ubuntu-latest + permissions: {} + concurrency: + group: "gh-aw-copilot-${{ github.workflow }}" + timeout-minutes: 10 + outputs: + success: ${{ steps.parse_results.outputs.success }} + steps: + - name: Setup Scripts + uses: github/gh-aw/actions/setup@48d8fdfddc8cad854ac0c70ceb573f09fb8f9c9b # v0.62.5 + with: + destination: /opt/gh-aw/actions + - name: Download agent artifacts + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: agent-artifacts + path: /tmp/gh-aw/threat-detection/ + - name: Download agent output artifact + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: agent-output + path: /tmp/gh-aw/threat-detection/ + - name: Echo agent output types + env: + AGENT_OUTPUT_TYPES: ${{ needs.agent.outputs.output_types }} + run: | + echo "Agent output-types: $AGENT_OUTPUT_TYPES" + - name: Setup threat detection + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + WORKFLOW_NAME: "Upgrade Python Library" + WORKFLOW_DESCRIPTION: "Pick an out-of-sync Python library from the todo list and upgrade it\nby running `scripts/update_lib quick`, then open a pull request." + HAS_PATCH: ${{ needs.agent.outputs.has_patch }} + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/setup_threat_detection.cjs'); + await main(); + - name: Ensure threat-detection directory and log + run: | + mkdir -p /tmp/gh-aw/threat-detection + touch /tmp/gh-aw/threat-detection/detection.log + - name: Validate COPILOT_GITHUB_TOKEN secret + id: validate-secret + run: /opt/gh-aw/actions/validate_multi_secret.sh COPILOT_GITHUB_TOKEN 'GitHub Copilot CLI' https://github.github.com/gh-aw/reference/engines/#github-copilot-default + env: + COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + - name: Install GitHub Copilot CLI + run: /opt/gh-aw/actions/install_copilot_cli.sh 0.0.409 + - name: Execute GitHub Copilot CLI + id: agentic_execution + # Copilot CLI tool arguments (sorted): + # --allow-tool shell(cat) + # --allow-tool shell(grep) + # --allow-tool shell(head) + # --allow-tool shell(jq) + # --allow-tool shell(ls) + # --allow-tool shell(tail) + # --allow-tool shell(wc) + timeout-minutes: 20 + run: | + set -o pipefail + COPILOT_CLI_INSTRUCTION="$(cat /tmp/gh-aw/aw-prompts/prompt.txt)" + mkdir -p /tmp/ + mkdir -p /tmp/gh-aw/ + mkdir -p /tmp/gh-aw/agent/ + mkdir -p /tmp/gh-aw/sandbox/agent/logs/ + copilot --add-dir /tmp/ --add-dir /tmp/gh-aw/ --add-dir /tmp/gh-aw/agent/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --allow-tool 'shell(cat)' --allow-tool 'shell(grep)' --allow-tool 'shell(head)' --allow-tool 'shell(jq)' --allow-tool 'shell(ls)' --allow-tool 'shell(tail)' --allow-tool 'shell(wc)' --share /tmp/gh-aw/sandbox/agent/logs/conversation.md --prompt "$COPILOT_CLI_INSTRUCTION"${GH_AW_MODEL_DETECTION_COPILOT:+ --model "$GH_AW_MODEL_DETECTION_COPILOT"} 2>&1 | tee /tmp/gh-aw/threat-detection/detection.log + env: + COPILOT_AGENT_RUNNER_TYPE: STANDALONE + COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + GH_AW_MODEL_DETECTION_COPILOT: ${{ vars.GH_AW_MODEL_DETECTION_COPILOT || '' }} + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GITHUB_HEAD_REF: ${{ github.head_ref }} + GITHUB_REF_NAME: ${{ github.ref_name }} + GITHUB_STEP_SUMMARY: ${{ env.GITHUB_STEP_SUMMARY }} + GITHUB_WORKSPACE: ${{ github.workspace }} + XDG_CONFIG_HOME: /home/runner + - name: Parse threat detection results + id: parse_results + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/parse_threat_detection_results.cjs'); + await main(); + - name: Upload threat detection log + if: always() + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: threat-detection.log + path: /tmp/gh-aw/threat-detection/detection.log + if-no-files-found: ignore + + safe_outputs: + needs: + - activation + - agent + - detection + if: ((!cancelled()) && (needs.agent.result != 'skipped')) && (needs.detection.outputs.success == 'true') + runs-on: ubuntu-slim + permissions: + contents: write + issues: write + pull-requests: write + timeout-minutes: 15 + env: + GH_AW_ENGINE_ID: "copilot" + GH_AW_WORKFLOW_ID: "upgrade-pylib" + GH_AW_WORKFLOW_NAME: "Upgrade Python Library" + outputs: + create_discussion_error_count: ${{ steps.process_safe_outputs.outputs.create_discussion_error_count }} + create_discussion_errors: ${{ steps.process_safe_outputs.outputs.create_discussion_errors }} + process_safe_outputs_processed_count: ${{ steps.process_safe_outputs.outputs.processed_count }} + process_safe_outputs_temporary_id_map: ${{ steps.process_safe_outputs.outputs.temporary_id_map }} + steps: + - name: Setup Scripts + uses: github/gh-aw/actions/setup@48d8fdfddc8cad854ac0c70ceb573f09fb8f9c9b # v0.62.5 + with: + destination: /opt/gh-aw/actions + - name: Download agent output artifact + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: agent-output + path: /tmp/gh-aw/safeoutputs/ + - name: Setup agent output environment variable + run: | + mkdir -p /tmp/gh-aw/safeoutputs/ + find "/tmp/gh-aw/safeoutputs/" -type f -print + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/safeoutputs/agent_output.json" >> "$GITHUB_ENV" + - name: Download patch artifact + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: agent-artifacts + path: /tmp/gh-aw/ + - name: Checkout repository + if: ((!cancelled()) && (needs.agent.result != 'skipped')) && (contains(needs.agent.outputs.output_types, 'create_pull_request')) + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + token: ${{ github.token }} + persist-credentials: false + fetch-depth: 1 + - name: Configure Git credentials + if: ((!cancelled()) && (needs.agent.result != 'skipped')) && (contains(needs.agent.outputs.output_types, 'create_pull_request')) + env: + REPO_NAME: ${{ github.repository }} + SERVER_URL: ${{ github.server_url }} + GIT_TOKEN: ${{ github.token }} + run: | + git config --global user.email "github-actions[bot]@users.noreply.github.com" + git config --global user.name "github-actions[bot]" + # Re-authenticate git with GitHub token + SERVER_URL_STRIPPED="${SERVER_URL#https://}" + git remote set-url origin "https://x-access-token:${GIT_TOKEN}@${SERVER_URL_STRIPPED}/${REPO_NAME}.git" + echo "Git configured with standard GitHub Actions identity" + - name: Process Safe Outputs + id: process_safe_outputs + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} + GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"create_pull_request\":{\"base_branch\":\"${{ github.ref_name }}\",\"draft\":false,\"expires\":30,\"labels\":[\"pylib-sync\"],\"max\":1,\"max_patch_size\":1024,\"title_prefix\":\"Update \"},\"missing_data\":{},\"missing_tool\":{}}" + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/safe_output_handler_manager.cjs'); + await main(); + diff --git a/.github/workflows/upgrade-pylib.md b/.github/workflows/upgrade-pylib.md new file mode 100644 index 00000000000..058fc067493 --- /dev/null +++ b/.github/workflows/upgrade-pylib.md @@ -0,0 +1,134 @@ +--- +description: | + Pick an out-of-sync Python library from the todo list and upgrade it + by running `scripts/update_lib quick`, then open a pull request. + +on: + workflow_dispatch: + inputs: + name: + description: "Module name to upgrade (leave empty to auto-pick)" + required: false + type: string + +timeout-minutes: 45 + +permissions: + contents: read + issues: read + pull-requests: read + +network: + allowed: + - defaults + - rust + - python + +engine: copilot + +runtimes: + python: + version: "3.14" + +tools: + bash: + - ":*" + edit: + github: + toolsets: [repos, issues, pull_requests] + read-only: true + +safe-outputs: + create-pull-request: + title-prefix: "Update " + labels: [pylib-sync] + draft: false + expires: 30 + +cache: + key: cpython-lib-${{ env.PYTHON_VERSION }} + path: cpython + restore-keys: + - cpython-lib- + +env: + PYTHON_VERSION: "v3.14.4" + ISSUE_ID: "6839" +--- + +# Upgrade Python Library + +You are an automated maintenance agent for RustPython, a Python 3 interpreter written in Rust. Your task is to upgrade one out-of-sync Python standard library module from CPython. + +## Step 1: Set up the environment + +The CPython source may already be cached. Check if the `cpython` directory exists and has the correct version: + +```bash +if [ -d "cpython/Lib" ]; then + echo "CPython cache hit, skipping clone" +else + git clone --depth 1 --branch "$PYTHON_VERSION" https://github.com/python/cpython.git cpython +fi +``` + +## Step 2: Determine module name + +Run this script to determine the module name: + +```bash +MODULE_NAME="${{ github.event.inputs.name }}" +if [ -z "$MODULE_NAME" ]; then + echo "No module specified, running todo to find one..." + python3 scripts/update_lib todo + echo "Pick one module from the list above that is marked [ ], has no unmet deps, and has a small Δ number." + echo "Do NOT pick: opcode, datetime, random, hashlib, tokenize, pdb, _pyrepl, concurrent, asyncio, multiprocessing, ctypes, idlelib, tkinter, shutil, tarfile, email, unittest" +else + echo "Module specified by user: $MODULE_NAME" +fi +``` + +If the script printed "Module specified by user: ...", use that exact name. If it printed the todo list, pick one suitable module from it. + +## Step 3: Run the upgrade + +Run the quick upgrade command. This will copy the library from CPython, migrate test files preserving RustPython markers, auto-mark test failures, and create a git commit: + +```bash +python3 scripts/update_lib quick +``` + +This takes a while because it builds RustPython (`cargo build --release`) and runs tests to determine which ones pass or fail. + +If the command fails, report the error and stop. Do not try to fix Rust code or modify test files manually. + +## Step 4: Verify the result + +After the script succeeds, check what changed: + +```bash +git log -1 --stat +git diff HEAD~1 --stat +``` + +Make sure the commit was created with the correct message format: `Update from `. + +## Step 5: Create the pull request + +Create a pull request. Reference issue #${{ env.ISSUE_ID }} in the body but do **NOT** use keywords that auto-close issues (Fix, Close, Resolve). + +Use this format for the PR body: + +``` +## Summary + +Upgrade `` from CPython $PYTHON_VERSION. + +Part of #$ISSUE_ID + +## Changes + +- Updated `Lib/` from CPython +- Migrated test files preserving RustPython markers +- Auto-marked test failures with `@expectedFailure` +``` diff --git a/.github/zizmor.yml b/.github/zizmor.yml new file mode 100644 index 00000000000..f22f76b70d8 --- /dev/null +++ b/.github/zizmor.yml @@ -0,0 +1,14 @@ +rules: + unpinned-uses: + config: + policies: + # dtolnay/rust-toolchain is a trusted action that uses lightweight branch + # refs (@stable, @nightly, etc.) by design. Pinning to a hash would break + # the intended usage pattern. + # We can remove this once https://github.com/dtolnay/rust-toolchain/issues/180 is resolved + dtolnay/rust-toolchain: any + # dtolnay/rust-toolchain handles component installation, target addition, and + # override configuration beyond what a bare `rustup` invocation provides. + # See: https://github.com/zizmorcore/zizmor/issues/1817 + superfluous-actions: + disable: true diff --git a/.gitignore b/.gitignore index 20d3d47d059..338a6437ca2 100644 --- a/.gitignore +++ b/.gitignore @@ -10,7 +10,6 @@ __pycache__/ wasm-pack.log .idea/ .envrc -.python-version flame-graph.html flame.txt @@ -22,8 +21,10 @@ flamescope.json extra_tests/snippets/resources extra_tests/not_impl.py +Lib/_sysconfig_vars*.json Lib/site-packages/* !Lib/site-packages/README.txt Lib/test/data/* !Lib/test/data/README cpython/ + diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000000..00a0a194b87 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,71 @@ +# NOTE: Reason for not using `prek.toml` is dependabot supports `pre-commit` as an ecosystem +# See: https://github.blog/changelog/2026-03-10-dependabot-now-supports-pre-commit-hooks/ + +fail_fast: false +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v6.0.0 + hooks: + - id: check-merge-conflict + priority: 0 + + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.15.7 + hooks: + - id: ruff-format + priority: 0 + + - id: ruff-check + args: [--select, I, --fix, --exit-non-zero-on-fix] + types_or: [python] + require_serial: true + priority: 1 + + - repo: local + hooks: + - id: redundant-test-patches + name: check redundant test patches + entry: scripts/check_redundant_patches.py + files: '^Lib/test/.*\.py$' + language: script + types: [python] + priority: 0 + + - repo: local + hooks: + - id: rustfmt + name: rustfmt + entry: rustfmt + language: system + types: [rust] + priority: 0 + + - id: generate-opcode-metadata + name: generate opcode metadata + entry: python scripts/generate_opcode_metadata.py + files: '^(crates/compiler-core/src/bytecode/instruction\.rs|scripts/generate_opcode_metadata\.py)$' + pass_filenames: false + language: system + require_serial: true + priority: 1 # so rustfmt runs first + + - repo: https://github.com/streetsidesoftware/cspell-cli + rev: v9.7.0 + hooks: + - id: cspell + types: [rust] + additional_dependencies: + - '@cspell/dict-en_us' + - '@cspell/dict-cpp' + - '@cspell/dict-python' + - '@cspell/dict-rust' + - '@cspell/dict-win32' + - '@cspell/dict-shell' + priority: 0 + + - repo: https://github.com/rbubley/mirrors-prettier + rev: v3.8.1 + hooks: + - id: prettier + files: '^wasm/.*$' + priority: 0 diff --git a/.python-version b/.python-version new file mode 100644 index 00000000000..0104088a93f --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.14.4 diff --git a/AGENTS.md b/AGENTS.md index fa14977953a..b407328cffb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -93,7 +93,7 @@ rm -r target/debug/build/rustpython-* && find . | grep -E "\.pyc$" | xargs rm -r ```bash # Run Rust unit tests -cargo test --workspace --exclude rustpython_wasm +cargo test --workspace --exclude rustpython_wasm --exclude rustpython-venvlauncher # Run Python snippets tests (debug mode recommended for faster compilation) cargo run -- extra_tests/snippets/builtin_bytes.py @@ -125,6 +125,36 @@ Run `./scripts/whats_left.py` to get a list of unimplemented methods, which is h - Follow Rust best practices for error handling and memory management - Use the macro system (`pyclass`, `pymodule`, `pyfunction`, etc.) when implementing Python functionality in Rust +#### Comments + +- Do not delete or rewrite existing comments unless they are factually wrong or directly contradict the new code. +- Do not add decorative section separators (e.g. `// -----------`, `// ===`, `/* *** */`). Use `///` doc-comments or short `//` comments only when they add value. + +#### Avoid Duplicate Code in Branches + +When branches differ only in a value but share common logic, extract the differing value first, then call the common logic once. + +**Bad:** +```rust +let result = if condition { + let msg = format!("message A: {x}"); + some_function(msg, shared_arg) +} else { + let msg = format!("message B"); + some_function(msg, shared_arg) +}; +``` + +**Good:** +```rust +let msg = if condition { + format!("message A: {x}") +} else { + format!("message B") +}; +let result = some_function(msg, shared_arg); +``` + ### Python Code - **IMPORTANT**: In most cases, Python code should not be edited. Bug fixes should be made through Rust code modifications only @@ -197,6 +227,10 @@ cargo build --target wasm32-wasip1 --no-default-features --features freeze-stdli cargo run --features jit ``` +### Linux Build and Debug on macOS + +See the "Testing on Linux from macOS" section in [DEVELOPMENT.md](DEVELOPMENT.md#testing-on-linux-from-macos). + ### Building venvlauncher (Windows) See DEVELOPMENT.md "CPython Version Upgrade Checklist" section. diff --git a/Cargo.lock b/Cargo.lock index c6462e235f3..20336689a32 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -79,9 +79,9 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstream" -version = "0.6.21" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" dependencies = [ "anstyle", "anstyle-parse", @@ -100,9 +100,9 @@ checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" [[package]] name = "anstyle-parse" -version = "0.2.7" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" dependencies = [ "utf8parse", ] @@ -249,9 +249,9 @@ checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "aws-lc-fips-sys" -version = "0.13.11" +version = "0.13.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df6ea8e07e2df15b9f09f2ac5ee2977369b06d116f0c4eb5fa4ad443b73c7f53" +checksum = "f8bce4948d2520386c6d92a6ea2d472300257702242e5a1d01d6add52bd2e7c1" dependencies = [ "bindgen 0.72.1", "cc", @@ -263,9 +263,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.15.4" +version = "1.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b7b6141e96a8c160799cc2d5adecd5cbbe5054cb8c7c4af53da0f83bb7ad256" +checksum = "a054912289d18629dc78375ba2c3726a3afe3ff71b4edba9dedfca0e3446d1fc" dependencies = [ "aws-lc-fips-sys", "aws-lc-sys", @@ -275,9 +275,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.37.0" +version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c34dda4df7017c8db52132f0f8a2e0f8161649d15723ed63fc00c82d0f2081a" +checksum = "1fa7e52a4c5c547c741610a2c6f123f3881e409b714cd27e6798ef020c514f0a" dependencies = [ "cc", "cmake", @@ -303,7 +303,7 @@ version = "0.71.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f58bf3d7db68cfbac37cfc485a8d711e87e064c3d0fe0435b92f7a407f9d6b3" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", "cexpr", "clang-sys", "itertools 0.13.0", @@ -323,7 +323,7 @@ version = "0.72.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", "cexpr", "clang-sys", "itertools 0.13.0", @@ -345,9 +345,21 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.10.0" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" +checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" + +[[package]] +name = "bitflagset" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64b6ee310aa7af14142c8c9121775774ff601ae055ed98ba7fac96098bcde1b9" +dependencies = [ + "num-integer", + "num-traits", + "radium", + "ref-cast", +] [[package]] name = "blake2" @@ -389,9 +401,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.19.1" +version = "3.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dd9dc738b7a8311c7ade152424974d8115f2cdad61e8dab8dac9f2362298510" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" dependencies = [ "allocator-api2", ] @@ -404,9 +416,9 @@ checksum = "1fbdf580320f38b612e485521afda1ee26d10cc9884efaaa750d383e13e3c5f4" [[package]] name = "bytes" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" [[package]] name = "bzip2" @@ -491,9 +503,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chrono" -version = "0.4.43" +version = "0.4.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fac4744fb15ae8337dc853fee7fb3f4e48c0fbaa23d0afe49c447b4fab126118" +checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" dependencies = [ "iana-time-zone", "js-sys", @@ -593,15 +605,6 @@ dependencies = [ "cc", ] -[[package]] -name = "cobs" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1" -dependencies = [ - "thiserror 2.0.18", -] - [[package]] name = "collection_literals" version = "1.0.3" @@ -709,9 +712,9 @@ dependencies = [ [[package]] name = "cranelift" -version = "0.128.1" +version = "0.130.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d483a248b5d971d1ef6a814385502a38d8dde8fbf08b4ad08b78c53b8d66f923" +checksum = "f13b593d4c3fe30bdf7e7bf4cbe78637849515822d305da6080c7ddda554d251" dependencies = [ "cranelift-codegen", "cranelift-frontend", @@ -720,46 +723,46 @@ dependencies = [ [[package]] name = "cranelift-assembler-x64" -version = "0.128.1" +version = "0.130.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d32b9105ce689b3e79ae288f62e9c2d0de66e4869176a11829e5c696da0f018f" +checksum = "4f248321c6a7d4de5dcf2939368e96a397ad3f53b6a076e38d0104d1da326d37" dependencies = [ "cranelift-assembler-x64-meta", ] [[package]] name = "cranelift-assembler-x64-meta" -version = "0.128.1" +version = "0.130.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e950e8dd96c1760f1c3a2b06d3d35584a3617239d034e73593ec096a1f3ea69" +checksum = "ab6d78ff1f7d9bf8b7e1afbedbf78ba49e38e9da479d4c8a2db094e22f64e2bc" dependencies = [ "cranelift-srcgen", ] [[package]] name = "cranelift-bforest" -version = "0.128.1" +version = "0.130.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d769576bc48246fccf7f07173739e5f7a7fb3270eb9ac363c0792cad8963c034" +checksum = "6b6005ba640213a5b95382aeaf6b82bf028309581c8d7349778d66f27dc1180b" dependencies = [ "cranelift-entity", + "wasmtime-internal-core", ] [[package]] name = "cranelift-bitset" -version = "0.128.1" +version = "0.130.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94d37c4589e52def48bd745c3b28b523d66ade8b074644ed3a366144c225f212" +checksum = "81fb5b134a12b559ff0c0f5af0fcd755ad380723b5016c4e0d36f74d39485340" dependencies = [ - "serde", - "serde_derive", + "wasmtime-internal-core", ] [[package]] name = "cranelift-codegen" -version = "0.128.1" +version = "0.130.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c23b5ab93367eba82bddf49b63d841d8a0b8b39fb89d82829de6647b3a747108" +checksum = "85837de8be7f17a4034a6b08816f05a3144345d2091937b39d415990daca28f4" dependencies = [ "bumpalo", "cranelift-assembler-x64", @@ -771,21 +774,22 @@ dependencies = [ "cranelift-entity", "cranelift-isle", "gimli", - "hashbrown 0.15.5", + "hashbrown 0.16.1", + "libm", "log", "regalloc2", "rustc-hash", "serde", "smallvec", "target-lexicon", - "wasmtime-internal-math", + "wasmtime-internal-core", ] [[package]] name = "cranelift-codegen-meta" -version = "0.128.1" +version = "0.130.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c6118d26dd046455d31374b9432947ea2ba445c21fd8724370edd072f51f3bd" +checksum = "e433faa87d38e5b8ff469e44a26fea4f93e58abd7a7c10bad9810056139700c9" dependencies = [ "cranelift-assembler-x64-meta", "cranelift-codegen-shared", @@ -795,35 +799,34 @@ dependencies = [ [[package]] name = "cranelift-codegen-shared" -version = "0.128.1" +version = "0.130.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a068c67f04f37de835fda87a10491e266eea9f9283d0887d8bd0a2c0726588a9" +checksum = "5397ba61976e13944ca71230775db13ee1cb62849701ed35b753f4761ed0a9b7" [[package]] name = "cranelift-control" -version = "0.128.1" +version = "0.130.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35ceb830549fcd7f05493a3b6d3d2bcfa4d43588b099e8c2393d2d140d6f7951" +checksum = "cc81c88765580720eb30f4fc2c1bfdb75fcbf3094f87b3cd69cecca79d77a245" dependencies = [ "arbitrary", ] [[package]] name = "cranelift-entity" -version = "0.128.1" +version = "0.130.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b130f0edd119e7665f1875b8d686bd3fccefd9d74d10e9005cbcd76392e1831" +checksum = "463feed5d46cf8763f3ba3045284cf706dd161496e20ec9c14afbb4ba09b9e66" dependencies = [ "cranelift-bitset", - "serde", - "serde_derive", + "wasmtime-internal-core", ] [[package]] name = "cranelift-frontend" -version = "0.128.1" +version = "0.130.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "626a46aa207183bae011de3411a40951c494cea3fb2ef223d3118f75e13b23ca" +checksum = "a4c5eca7696c1c04ab4c7ed8d18eadbb47d6cc9f14ec86fe0881bf1d7e97e261" dependencies = [ "cranelift-codegen", "log", @@ -833,15 +836,15 @@ dependencies = [ [[package]] name = "cranelift-isle" -version = "0.128.1" +version = "0.130.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d09dab08a5129cf59919fdd4567e599ea955de62191a852982150ac42ce4ab21" +checksum = "f1153844610cc9c6da8cf10ce205e45da1a585b7688ed558aa808bbe2e4e6d77" [[package]] name = "cranelift-jit" -version = "0.128.1" +version = "0.130.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aaab95b37e712267c51ca968ed4fa83d1a79b9ff3bc86fb9469c764340f486e4" +checksum = "41836de8321b303d3d4188e58cc09c30c7645337342acfcfb363732695cae098" dependencies = [ "anyhow", "cranelift-codegen", @@ -859,9 +862,9 @@ dependencies = [ [[package]] name = "cranelift-module" -version = "0.128.1" +version = "0.130.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d53f2d6b64ef9fb21da36698d45715639e0df50224883baa1e9bd04f96f0716" +checksum = "b731f66cb1b69b60a74216e632968ebdbb95c488d26aa1448ec226ae0ffec33e" dependencies = [ "anyhow", "cranelift-codegen", @@ -870,9 +873,9 @@ dependencies = [ [[package]] name = "cranelift-native" -version = "0.128.1" +version = "0.130.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "847b8eaef0f7095b401d3ce80587036495b94e7a051904df9e28d6cd14e69b94" +checksum = "a97b583fe9a60f06b0464cee6be5a17f623fd91b217aaac99b51b339d19911af" dependencies = [ "cranelift-codegen", "libc", @@ -881,9 +884,9 @@ dependencies = [ [[package]] name = "cranelift-srcgen" -version = "0.128.1" +version = "0.130.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15a4849e90e778f2fcc9fd1b93bd074dbf6b8b6f420951f9617c4774fe71e7fc" +checksum = "8594dc6bb4860fa8292f1814c76459dbfb933e1978d8222de6380efce45c7cee" [[package]] name = "crc32fast" @@ -896,9 +899,9 @@ dependencies = [ [[package]] name = "criterion" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d883447757bb0ee46f233e9dc22eb84d93a9508c9b868687b274fc431d886bf" +checksum = "950046b2aa2492f9a536f5f4f9a3de7b9e2476e575e05bd6c333371add4d98f3" dependencies = [ "alloca", "anes", @@ -921,9 +924,9 @@ dependencies = [ [[package]] name = "criterion-plot" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed943f81ea2faa8dcecbbfa50164acf95d555afec96a27871663b300e387b2e4" +checksum = "d8d80a2f4f5b554395e47b5d8305bc3d27813bacb73493eb1001e8f76dae29ea" dependencies = [ "cast", "itertools 0.13.0", @@ -1116,18 +1119,6 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" -[[package]] -name = "embedded-io" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" - -[[package]] -name = "embedded-io" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" - [[package]] name = "encode_unicode" version = "1.0.0" @@ -1142,25 +1133,19 @@ checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" [[package]] name = "env_filter" -version = "0.1.4" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bf3c259d255ca70051b30e2e95b5446cdb8949ac4cd22c0d7fd634d89f568e2" +checksum = "7a1c3cc8e57274ec99de65301228b537f1e4eedc1b8e0f9411c6caac8ae7308f" dependencies = [ "log", "regex", ] -[[package]] -name = "env_home" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7f84e12ccf0a7ddc17a6c41c93326024c42920d7ee630d04950e6926645c0fe" - [[package]] name = "env_logger" -version = "0.11.8" +version = "0.11.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c863f0904021b108aa8b2f55046443e6b1ebde8fd4a15c399893aae4fa069f" +checksum = "0621c04f2196ac3f488dd583365b9c09be011a4ab8b9f37248ffcc8f6198b56a" dependencies = [ "anstream", "anstyle", @@ -1197,12 +1182,6 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de853764b47027c2e862a995c34978ffa63c1501f2e15f987ba11bd4f9bba193" -[[package]] -name = "fallible-iterator" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" - [[package]] name = "fastrand" version = "2.3.0" @@ -1270,20 +1249,19 @@ dependencies = [ [[package]] name = "flate2" -version = "1.1.8" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b375d6465b98090a5f25b1c7703f3859783755aa9a80433b36e0379a3ec2f369" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" dependencies = [ - "crc32fast", "miniz_oxide", - "zlib-rs", + "zlib-rs 0.6.0", ] [[package]] -name = "foldhash" -version = "0.1.5" +name = "fnv" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "foldhash" @@ -1417,11 +1395,12 @@ dependencies = [ [[package]] name = "gimli" -version = "0.32.3" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" +checksum = "0bf7f043f89559805f8c7cacc432749b2fa0d0a0a9ee46ce47164ed5ba7f126c" dependencies = [ - "fallible-iterator", + "fnv", + "hashbrown 0.16.1", "indexmap", "stable_deref_trait", ] @@ -1448,10 +1427,6 @@ name = "hashbrown" version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" -dependencies = [ - "foldhash 0.1.5", - "serde", -] [[package]] name = "hashbrown" @@ -1459,7 +1434,7 @@ version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" dependencies = [ - "foldhash 0.2.0", + "foldhash", ] [[package]] @@ -1529,24 +1504,97 @@ dependencies = [ ] [[package]] -name = "indexmap" -version = "2.13.0" +name = "icu_collections" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" dependencies = [ - "equivalent", - "hashbrown 0.16.1", - "serde", - "serde_core", + "displaydoc", + "potential_utf", + "yoke", + "zerofrom", + "zerovec", ] [[package]] -name = "indoc" -version = "2.0.7" +name = "icu_locale_core" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706" +checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" dependencies = [ - "rustversion", + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "write16", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" + +[[package]] +name = "icu_properties" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" + +[[package]] +name = "icu_provider" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "indexmap" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +dependencies = [ + "equivalent", + "hashbrown 0.16.1", ] [[package]] @@ -1561,9 +1609,9 @@ dependencies = [ [[package]] name = "insta" -version = "1.46.1" +version = "1.46.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "248b42847813a1550dafd15296fd9748c651d0c32194559dbc05d804d54b21e8" +checksum = "e82db8c87c7f1ccecb34ce0c24399b8a73081427f3c7c50a5d597925356115e4" dependencies = [ "console", "once_cell", @@ -1621,9 +1669,9 @@ checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" [[package]] name = "jiff" -version = "0.2.18" +version = "0.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e67e8da4c49d6d9909fe03361f9b620f58898859f5c7aded68351e85e71ecf50" +checksum = "1a3546dc96b6d42c5f24902af9e2538e82e39ad350b0c766eb3fbf2d8f3d8359" dependencies = [ "jiff-static", "log", @@ -1634,9 +1682,9 @@ dependencies = [ [[package]] name = "jiff-static" -version = "0.2.18" +version = "0.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0c84ee7f197eca9a86c6fd6cb771e55eb991632f15f2bc3ca6ec838929e6e78" +checksum = "2a8c8b344124222efd714b73bb41f8b5120b27a7cc1c75593a6ff768d9d05aa4" dependencies = [ "proc-macro2", "quote", @@ -1687,9 +1735,9 @@ dependencies = [ [[package]] name = "junction" -version = "1.4.1" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "642883fdc81cf2da15ee8183fa1d2c7da452414dd41541a0f3e1428069345447" +checksum = "8cfc352a66ba903c23239ef51e809508b6fc2b0f90e3476ac7a9ff47e863ae95" dependencies = [ "scopeguard", "windows-sys 0.61.2", @@ -1697,9 +1745,9 @@ dependencies = [ [[package]] name = "keccak" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecc2af9a1119c51f12a14607e783cb977bde58bc069ff0c3da1095e635d70654" +checksum = "cb26cec98cce3a3d96cbb7bced3c4b16e3d13f27ec56dbd62cbc8f39cfb9d653" dependencies = [ "cpufeatures", ] @@ -1743,9 +1791,9 @@ checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" [[package]] name = "lexopt" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fa0e2a1fcbe2f6be6c42e342259976206b383122fc152e872795338b5a3f3a7" +checksum = "803ec87c9cfb29b9d2633f20cba1f488db3fd53f2158b1024cbefb47ba05d413" [[package]] name = "libbz2-rs-sys" @@ -1755,9 +1803,9 @@ checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7" [[package]] name = "libc" -version = "0.2.180" +version = "0.2.183" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcc35a38544a891a5f7c865aca548a982ccb3b8650a5b06d0fd33a10283c56fc" +checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" [[package]] name = "libffi" @@ -1798,6 +1846,26 @@ dependencies = [ "windows-link", ] +[[package]] +name = "liblzma" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6033b77c21d1f56deeae8014eb9fbe7bdf1765185a6c508b5ca82eeaed7f899" +dependencies = [ + "liblzma-sys", +] + +[[package]] +name = "liblzma-sys" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f2db66f3268487b5033077f266da6777d057949b8f93c8ad82e441df25e6186" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "libm" version = "0.2.16" @@ -1810,15 +1878,15 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d0b95e02c851351f877147b7deea7b1afb1df71b63aa5f8270716e0c5720616" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", "libc", ] [[package]] name = "libsqlite3-sys" -version = "0.36.0" +version = "0.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95b4103cffefa72eb8428cb6b47d6627161e51c2739fc5e3b734584157bc642a" +checksum = "b1f111c8c41e7c61a49cd34e44c7619462967221a6443b0ec299e0ac30cfb9b1" dependencies = [ "cc", "pkg-config", @@ -1831,14 +1899,20 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c10501e7805cee23da17c7790e59df2870c0d4043ec6d03f67d31e2b53e77415" dependencies = [ - "zlib-rs", + "zlib-rs 0.5.5", ] [[package]] name = "linux-raw-sys" -version = "0.11.0" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" +checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" [[package]] name = "lock_api" @@ -1857,24 +1931,13 @@ checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" [[package]] name = "lz4_flex" -version = "0.12.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab6473172471198271ff72e9379150e9dfd70d8e533e0752a27e515b48dd375e" +checksum = "db9a0d582c2874f68138a16ce1867e0ffde6c0bb0a0df85e1f36d04146db488a" dependencies = [ "twox-hash", ] -[[package]] -name = "lzma-sys" -version = "0.1.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" -dependencies = [ - "cc", - "libc", - "pkg-config", -] - [[package]] name = "mac_address" version = "1.1.8" @@ -1896,9 +1959,9 @@ dependencies = [ [[package]] name = "malachite-base" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5eb2a098b227df48779e28ed4125dd3161b792a9254961377cea6f5c19e5b417" +checksum = "a8b6f86fdbb1eb9955946be91775239dfcb0acdb1a51bb07d5fc9b8c854f5ccd" dependencies = [ "hashbrown 0.16.1", "itertools 0.14.0", @@ -1908,9 +1971,9 @@ dependencies = [ [[package]] name = "malachite-bigint" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6eaf19f1b8ba023528050372eafd72ca11f80f70c9dc2af9bb22f888bf079013" +checksum = "67fcd6e504ffc67db2b3c6d5e90e08054646e2b04f42115a5460bf1c1e37d3bc" dependencies = [ "malachite-base", "malachite-nz", @@ -1921,9 +1984,9 @@ dependencies = [ [[package]] name = "malachite-nz" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab7c0ddc4e2681459d70591baf30ca5abd31c25969e76d3605838bec794c8077" +checksum = "0197a2f5cfee19d59178e282985c6ca79a9233e26a2adcf40acb693896aa09f6" dependencies = [ "itertools 0.14.0", "libm", @@ -1933,9 +1996,9 @@ dependencies = [ [[package]] name = "malachite-q" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13d19f04fc672f251d477c8d58c13c6c5550553dfba6a665c9ad3604466ac9a" +checksum = "be2add95162aede090c48f0ee51bea7d328847ce3180aa44588111f846cc116b" dependencies = [ "itertools 0.14.0", "malachite-base", @@ -1971,12 +2034,6 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" -[[package]] -name = "matches" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" - [[package]] name = "md-5" version = "0.10.6" @@ -1989,15 +2046,15 @@ dependencies = [ [[package]] name = "memchr" -version = "2.7.6" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" [[package]] name = "memmap2" -version = "0.9.9" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "744133e4a0e0a658e1374cf3bf8e415c4052a15a111acd372764c55b4177d490" +checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3" dependencies = [ "libc", ] @@ -2051,7 +2108,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", "cfg-if", "cfg_aliases", "libc", @@ -2064,7 +2121,7 @@ version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", "cfg-if", "cfg_aliases", "libc", @@ -2136,9 +2193,9 @@ dependencies = [ [[package]] name = "num_enum" -version = "0.7.5" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1207a7e20ad57b847bbddc6776b968420d38292bbfe2089accff5e19e82454c" +checksum = "5d0bca838442ec211fa11de3a8b0e0e8f3a4522575b5c4c06ed722e005036f26" dependencies = [ "num_enum_derive", "rustversion", @@ -2146,9 +2203,9 @@ dependencies = [ [[package]] name = "num_enum_derive" -version = "0.7.5" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff32365de1b6743cb203b710788263c44a03de03802daf96092f2da4fe6ba4d7" +checksum = "680998035259dcfcafe653688bf2aa6d3e2dc05e98be6ab46afb089dc84f1df8" dependencies = [ "proc-macro2", "quote", @@ -2193,11 +2250,11 @@ checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" [[package]] name = "openssl" -version = "0.10.75" +version = "0.10.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08838db121398ad17ab8531ce9de97b244589089e290a384c900cb9ff7434328" +checksum = "951c002c75e16ea2c65b8c7e4d3d51d5530d8dfa7d060b4776828c88cfb18ecf" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", "cfg-if", "foreign-types", "libc", @@ -2234,9 +2291,9 @@ dependencies = [ [[package]] name = "openssl-sys" -version = "0.9.111" +version = "0.9.112" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82cab2d520aa75e3c58898289429321eb788c3106963d0dc886ec7a5f4adc321" +checksum = "57d55af3b3e226502be1526dfdba67ab0e9c96fc293004e79576b2b9edb0dbdb" dependencies = [ "cc", "libc", @@ -2283,8 +2340,7 @@ dependencies = [ [[package]] name = "parking_lot_core" version = "0.9.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +source = "git+https://github.com/youknowone/parking_lot?branch=rustpython#4392edbe879acc9c0dd94eda53d2205d3ab912c9" dependencies = [ "cfg-if", "libc", @@ -2508,15 +2564,12 @@ dependencies = [ ] [[package]] -name = "postcard" -version = "1.1.3" +name = "potential_utf" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6764c3b5dd454e283a30e6dfe78e9b31096d9e32036b5d1eaac7a6119ccb9a24" +checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" dependencies = [ - "cobs", - "embedded-io 0.4.0", - "embedded-io 0.6.1", - "serde", + "zerovec", ] [[package]] @@ -2566,9 +2619,9 @@ dependencies = [ [[package]] name = "psm" -version = "0.1.29" +version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa96cb91275ed31d6da3e983447320c4eb219ac180fa1679a0889ff32861e2d" +checksum = "3852766467df634d74f0b2d7819bf8dc483a0eb2e3b0f50f756f9cfe8b0d18d8" dependencies = [ "ar_archive_writer", "cc", @@ -2576,9 +2629,9 @@ dependencies = [ [[package]] name = "pymath" -version = "0.1.5" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbfb6723b732fc7f0b29a0ee7150c7f70f947bf467b8c3e82530b13589a78b4c" +checksum = "bc10e50b7a1f2cc3887e983721cb51fc7574be0066c84bff3ef9e5c096e8d6d5" dependencies = [ "libc", "libm", @@ -2590,35 +2643,32 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.27.2" +version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab53c047fcd1a1d2a8820fe84f05d6be69e9526be40cb03b73f86b6b03e6d87d" +checksum = "91fd8e38a3b50ed1167fb981cd6fd60147e091784c427b8f7183a7ee32c31c12" dependencies = [ - "indoc", "libc", - "memoffset", "once_cell", "portable-atomic", "pyo3-build-config", "pyo3-ffi", "pyo3-macros", - "unindent", ] [[package]] name = "pyo3-build-config" -version = "0.27.2" +version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b455933107de8642b4487ed26d912c2d899dec6114884214a0b3bb3be9261ea6" +checksum = "e368e7ddfdeb98c9bca7f8383be1648fd84ab466bf2bc015e94008db6d35611e" dependencies = [ "target-lexicon", ] [[package]] name = "pyo3-ffi" -version = "0.27.2" +version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c85c9cbfaddf651b1221594209aed57e9e5cff63c4d11d1feead529b872a089" +checksum = "7f29e10af80b1f7ccaf7f69eace800a03ecd13e883acfacc1e5d0988605f651e" dependencies = [ "libc", "pyo3-build-config", @@ -2626,9 +2676,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.27.2" +version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a5b10c9bf9888125d917fb4d2ca2d25c8df94c7ab5a52e13313a07e050a3b02" +checksum = "df6e520eff47c45997d2fc7dd8214b25dd1310918bbb2642156ef66a67f29813" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -2638,9 +2688,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.27.2" +version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03b51720d314836e53327f5871d4c0cfb4fb37cc2c4a11cc71907a86342c40f9" +checksum = "c4cdc218d835738f81c2338f822078af45b4afdf8b2e33cbb5916f108b813acb" dependencies = [ "heck", "proc-macro2", @@ -2651,9 +2701,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.44" +version = "1.0.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" dependencies = [ "proc-macro2", ] @@ -2796,7 +2846,7 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", ] [[package]] @@ -2810,11 +2860,31 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "ref-cast" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "regalloc2" -version = "0.13.5" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08effbc1fa53aaebff69521a5c05640523fab037b34a4a2c109506bc938246fa" +checksum = "952ddbfc6f9f64d006c3efd8c9851a6ba2f2b944ba94730db255d55006e0ffda" dependencies = [ "allocator-api2", "bumpalo", @@ -2900,72 +2970,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "ruff_python_ast" -version = "0.0.0" -source = "git+https://github.com/astral-sh/ruff.git?rev=8b2e7b36f246b990fe473a84eef25ff429e59ecf#8b2e7b36f246b990fe473a84eef25ff429e59ecf" -dependencies = [ - "aho-corasick", - "bitflags 2.10.0", - "compact_str", - "get-size2", - "is-macro", - "memchr", - "ruff_python_trivia", - "ruff_source_file", - "ruff_text_size", - "rustc-hash", - "thiserror 2.0.18", -] - -[[package]] -name = "ruff_python_parser" -version = "0.0.0" -source = "git+https://github.com/astral-sh/ruff.git?rev=8b2e7b36f246b990fe473a84eef25ff429e59ecf#8b2e7b36f246b990fe473a84eef25ff429e59ecf" -dependencies = [ - "bitflags 2.10.0", - "bstr", - "compact_str", - "get-size2", - "memchr", - "ruff_python_ast", - "ruff_python_trivia", - "ruff_text_size", - "rustc-hash", - "static_assertions", - "unicode-ident", - "unicode-normalization", - "unicode_names2 1.3.0", -] - -[[package]] -name = "ruff_python_trivia" -version = "0.0.0" -source = "git+https://github.com/astral-sh/ruff.git?rev=8b2e7b36f246b990fe473a84eef25ff429e59ecf#8b2e7b36f246b990fe473a84eef25ff429e59ecf" -dependencies = [ - "itertools 0.14.0", - "ruff_source_file", - "ruff_text_size", - "unicode-ident", -] - -[[package]] -name = "ruff_source_file" -version = "0.0.0" -source = "git+https://github.com/astral-sh/ruff.git?rev=8b2e7b36f246b990fe473a84eef25ff429e59ecf#8b2e7b36f246b990fe473a84eef25ff429e59ecf" -dependencies = [ - "memchr", - "ruff_text_size", -] - -[[package]] -name = "ruff_text_size" -version = "0.0.0" -source = "git+https://github.com/astral-sh/ruff.git?rev=8b2e7b36f246b990fe473a84eef25ff429e59ecf#8b2e7b36f246b990fe473a84eef25ff429e59ecf" -dependencies = [ - "get-size2", -] - [[package]] name = "rustc-hash" version = "2.1.1" @@ -2983,11 +2987,11 @@ dependencies = [ [[package]] name = "rustix" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "146c9e247ccc180c1f61615433868c99f3de3ae256a30a43b49f67c2d9171f34" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", "errno", "libc", "linux-raw-sys", @@ -2996,9 +3000,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.36" +version = "0.23.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c665f33d38cea657d9614f766881e4d510e0eda4239891eea56b4cadcf01801b" +checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" dependencies = [ "aws-lc-rs", "once_cell", @@ -3067,9 +3071,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.103.9" +version = "0.103.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53" +checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef" dependencies = [ "aws-lc-rs", "ring", @@ -3079,7 +3083,7 @@ dependencies = [ [[package]] name = "rustpython" -version = "0.4.0" +version = "0.5.0" dependencies = [ "cfg-if", "criterion", @@ -3091,9 +3095,9 @@ dependencies = [ "libc", "log", "pyo3", - "ruff_python_parser", "rustpython-compiler", "rustpython-pylib", + "rustpython-ruff_python_parser", "rustpython-stdlib", "rustpython-vm", "rustyline", @@ -3102,10 +3106,10 @@ dependencies = [ [[package]] name = "rustpython-codegen" -version = "0.4.0" +version = "0.5.0" dependencies = [ "ahash", - "bitflags 2.10.0", + "bitflags 2.11.0", "indexmap", "insta", "itertools 0.14.0", @@ -3114,11 +3118,11 @@ dependencies = [ "memchr", "num-complex", "num-traits", - "ruff_python_ast", - "ruff_python_parser", - "ruff_text_size", "rustpython-compiler-core", "rustpython-literal", + "rustpython-ruff_python_ast", + "rustpython-ruff_python_parser", + "rustpython-ruff_text_size", "rustpython-wtf8", "thiserror 2.0.18", "unicode_names2 2.0.0", @@ -3126,10 +3130,10 @@ dependencies = [ [[package]] name = "rustpython-common" -version = "0.4.0" +version = "0.5.0" dependencies = [ "ascii", - "bitflags 2.10.0", + "bitflags 2.11.0", "cfg-if", "getrandom 0.3.4", "itertools 0.14.0", @@ -3141,7 +3145,6 @@ dependencies = [ "nix 0.30.1", "num-complex", "num-traits", - "once_cell", "parking_lot", "radium", "rustpython-literal", @@ -3154,41 +3157,42 @@ dependencies = [ [[package]] name = "rustpython-compiler" -version = "0.4.0" +version = "0.5.0" dependencies = [ - "ruff_python_ast", - "ruff_python_parser", - "ruff_source_file", - "ruff_text_size", "rustpython-codegen", "rustpython-compiler-core", + "rustpython-ruff_python_ast", + "rustpython-ruff_python_parser", + "rustpython-ruff_source_file", + "rustpython-ruff_text_size", "thiserror 2.0.18", ] [[package]] name = "rustpython-compiler-core" -version = "0.4.0" +version = "0.5.0" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", + "bitflagset", "itertools 0.14.0", "lz4_flex", "malachite-bigint", "num-complex", - "ruff_source_file", + "rustpython-ruff_source_file", "rustpython-wtf8", ] [[package]] name = "rustpython-compiler-source" -version = "0.5.0+deprecated" +version = "0.4.1+deprecated" dependencies = [ - "ruff_source_file", - "ruff_text_size", + "rustpython-ruff_source_file", + "rustpython-ruff_text_size", ] [[package]] name = "rustpython-derive" -version = "0.4.0" +version = "0.5.0" dependencies = [ "rustpython-compiler", "rustpython-derive-impl", @@ -3197,7 +3201,7 @@ dependencies = [ [[package]] name = "rustpython-derive-impl" -version = "0.4.0" +version = "0.5.0" dependencies = [ "itertools 0.14.0", "maplit", @@ -3212,14 +3216,14 @@ dependencies = [ [[package]] name = "rustpython-doc" -version = "0.4.0" +version = "0.5.0" dependencies = [ "phf 0.13.1", ] [[package]] name = "rustpython-jit" -version = "0.4.0" +version = "0.5.0" dependencies = [ "approx", "cranelift", @@ -3235,31 +3239,102 @@ dependencies = [ [[package]] name = "rustpython-literal" -version = "0.4.0" +version = "0.5.0" dependencies = [ "hexf-parse", + "icu_properties", "is-macro", "lexical-parse-float", "num-traits", "rand 0.9.2", "rustpython-wtf8", - "unic-ucd-category", ] [[package]] name = "rustpython-pylib" -version = "0.4.0" +version = "0.5.0" dependencies = [ "glob", "rustpython-compiler-core", "rustpython-derive", ] +[[package]] +name = "rustpython-ruff_python_ast" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f021ff72cabf5e2cd6d8ec8813d376a8445a228dc610ab56c27bd9054cda70d4" +dependencies = [ + "aho-corasick", + "bitflags 2.11.0", + "compact_str", + "get-size2", + "is-macro", + "memchr", + "rustc-hash", + "rustpython-ruff_python_trivia", + "rustpython-ruff_source_file", + "rustpython-ruff_text_size", + "thiserror 2.0.18", +] + +[[package]] +name = "rustpython-ruff_python_parser" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01e6ee78bd9671fb5766664b2695fe1f2a92a961f4d9101646c570d8acdb1e0b" +dependencies = [ + "bitflags 2.11.0", + "bstr", + "compact_str", + "get-size2", + "memchr", + "rustc-hash", + "rustpython-ruff_python_ast", + "rustpython-ruff_python_trivia", + "rustpython-ruff_text_size", + "static_assertions", + "unicode-ident", + "unicode-normalization", + "unicode_names2 1.3.0", +] + +[[package]] +name = "rustpython-ruff_python_trivia" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79e7cfd1056f3a02ff0d2d0e4474286ca963260782f878b7b81c1dd87432e682" +dependencies = [ + "itertools 0.14.0", + "rustpython-ruff_source_file", + "rustpython-ruff_text_size", + "unicode-ident", +] + +[[package]] +name = "rustpython-ruff_source_file" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "948107aad62ddb12a11fc7bf68a49e52a0b0a3737d415a2505e54f5a9edac737" +dependencies = [ + "memchr", + "rustpython-ruff_text_size", +] + +[[package]] +name = "rustpython-ruff_text_size" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8291ee0f5a779e54ccd4e0151a0c426f8b49a123f99b5b6545db17ccdd4277aa" +dependencies = [ + "get-size2", +] + [[package]] name = "rustpython-sre_engine" -version = "0.4.0" +version = "0.5.0" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", "criterion", "num_enum", "optional", @@ -3268,7 +3343,7 @@ dependencies = [ [[package]] name = "rustpython-stdlib" -version = "0.4.0" +version = "0.5.0" dependencies = [ "adler32", "ahash", @@ -3279,6 +3354,7 @@ dependencies = [ "bzip2", "cfg-if", "chrono", + "constant_time_eq", "crc32fast", "crossbeam-utils", "csv-core", @@ -3291,12 +3367,16 @@ dependencies = [ "foreign-types-shared", "gethostname", "hex", + "hmac", + "icu_normalizer", + "icu_properties", "indexmap", "itertools 0.14.0", "libc", + "liblzma", + "liblzma-sys", "libsqlite3-sys", "libz-rs-sys", - "lzma-sys", "mac_address", "malachite-bigint", "md-5", @@ -3305,7 +3385,6 @@ dependencies = [ "mt19937", "nix 0.30.1", "num-complex", - "num-integer", "num-traits", "num_enum", "oid-registry", @@ -3315,6 +3394,7 @@ dependencies = [ "page_size", "parking_lot", "paste", + "pbkdf2", "pem-rfc7468 1.0.0", "phf 0.13.1", "pkcs8", @@ -3327,6 +3407,10 @@ dependencies = [ "rustls-platform-verifier", "rustpython-common", "rustpython-derive", + "rustpython-ruff_python_ast", + "rustpython-ruff_python_parser", + "rustpython-ruff_source_file", + "rustpython-ruff_text_size", "rustpython-vm", "schannel", "sha-1", @@ -3338,14 +3422,7 @@ dependencies = [ "termios", "tk-sys", "ucd", - "unic-char-property", - "unic-normal", "unic-ucd-age", - "unic-ucd-bidi", - "unic-ucd-category", - "unic-ucd-ident", - "unicode-bidi-mirroring", - "unicode-casing", "unicode_names2 2.0.0", "uuid", "webpki-roots", @@ -3354,23 +3431,19 @@ dependencies = [ "x509-cert", "x509-parser", "xml", - "xz2", ] [[package]] name = "rustpython-venvlauncher" -version = "0.4.0" -dependencies = [ - "windows-sys 0.61.2", -] +version = "0.5.0" [[package]] name = "rustpython-vm" -version = "0.4.0" +version = "0.5.0" dependencies = [ "ahash", "ascii", - "bitflags 2.10.0", + "bitflags 2.11.0", "bstr", "caseless", "cfg-if", @@ -3385,6 +3458,7 @@ dependencies = [ "glob", "half", "hex", + "icu_properties", "indexmap", "is-macro", "itertools 0.14.0", @@ -3401,15 +3475,11 @@ dependencies = [ "num-traits", "num_cpus", "num_enum", - "once_cell", "optional", "parking_lot", "paste", "psm", "result-like", - "ruff_python_ast", - "ruff_python_parser", - "ruff_text_size", "rustix", "rustpython-codegen", "rustpython-common", @@ -3418,6 +3488,9 @@ dependencies = [ "rustpython-derive", "rustpython-jit", "rustpython-literal", + "rustpython-ruff_python_ast", + "rustpython-ruff_python_parser", + "rustpython-ruff_text_size", "rustpython-sre_engine", "rustyline", "scoped-tls", @@ -3427,12 +3500,8 @@ dependencies = [ "strum", "strum_macros", "thiserror 2.0.18", - "thread_local", "timsort", "uname", - "unic-ucd-bidi", - "unic-ucd-category", - "unic-ucd-ident", "unicode-casing", "wasm-bindgen", "which", @@ -3442,7 +3511,7 @@ dependencies = [ [[package]] name = "rustpython-wtf8" -version = "0.4.0" +version = "0.5.0" dependencies = [ "ascii", "bstr", @@ -3452,11 +3521,10 @@ dependencies = [ [[package]] name = "rustpython_wasm" -version = "0.4.0" +version = "0.5.0" dependencies = [ "console_error_panic_hook", "js-sys", - "ruff_python_parser", "rustpython-common", "rustpython-pylib", "rustpython-stdlib", @@ -3480,7 +3548,7 @@ version = "17.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e902948a25149d50edc1a8e0141aad50f54e22ba83ff988cf8f7c9ef07f50564" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", "cfg-if", "clipboard-win", "fd-lock", @@ -3504,9 +3572,9 @@ checksum = "a50f4cf475b65d88e057964e0e9bb1f0aa9bbb2036dc65c64596b42932536984" [[package]] name = "safe_arch" -version = "0.9.3" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "629516c85c29fe757770fa03f2074cf1eac43d44c02a3de9fc2ef7b0e207dfdd" +checksum = "1f7caad094bd561859bcd467734a720c3c1f5d1f338995351fefe2190c45efed" dependencies = [ "bytemuck", ] @@ -3531,9 +3599,9 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.28" +version = "0.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" dependencies = [ "windows-sys 0.61.2", ] @@ -3567,7 +3635,7 @@ version = "3.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3297343eaf830f66ede390ea39da1d462b6b0c1b000f420d0a83f898bbbe6ef" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", "core-foundation 0.10.1", "core-foundation-sys", "libc", @@ -3640,9 +3708,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "1.0.4" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8bbf91e5a4d6315eee45e704372590b30e260ee83af6639d64557f51b067776" +checksum = "876ac351060d4f882bb1032b6369eb0aef79ad9df1ea8bc404874d8cc3d0cd98" dependencies = [ "serde_core", ] @@ -3742,18 +3810,15 @@ name = "smallvec" version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" -dependencies = [ - "serde", -] [[package]] name = "socket2" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86f4aa3ad99f2088c990dfa82d367e19cb29268ed67c574d10d0a4bfe71f07e0" +checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -3780,15 +3845,15 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "strum" -version = "0.27.2" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" +checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd" [[package]] name = "strum_macros" -version = "0.27.2" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" +checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664" dependencies = [ "heck", "proc-macro2", @@ -3804,9 +3869,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.114" +version = "2.0.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4d107df263a3013ef9b1879b0df87d706ff80f65a86ea879bd9c31f9b307c2a" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" dependencies = [ "proc-macro2", "quote", @@ -3841,7 +3906,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", "core-foundation 0.9.4", "system-configuration-sys", ] @@ -3950,20 +4015,11 @@ dependencies = [ "winapi", ] -[[package]] -name = "thread_local" -version = "1.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" -dependencies = [ - "cfg-if", -] - [[package]] name = "time" -version = "0.3.46" +version = "0.3.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9da98b7d9b7dad93488a84b8248efc35352b0b2657397d4167e7ad67e5d535e5" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" dependencies = [ "deranged", "itoa", @@ -3982,9 +4038,9 @@ checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" [[package]] name = "time-macros" -version = "0.2.26" +version = "0.2.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78cc610bac2dcee56805c99642447d4c5dbde4d01f752ffea0199aee1f601dc4" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" dependencies = [ "num-conv", "time-core", @@ -3996,6 +4052,16 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "639ce8ef6d2ba56be0383a94dd13b92138d58de44c62618303bb798fa92bdc00" +[[package]] +name = "tinystr" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "tinytemplate" version = "1.2.1" @@ -4053,9 +4119,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.9.11+spec-1.1.0" +version = "1.1.0+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3afc9a848309fe1aaffaed6e1546a7a14de1f935dc9d89d32afd9a44bab7c46" +checksum = "f8195ca05e4eb728f4ba94f3e3291661320af739c4e43779cbdfae82ab239fcc" dependencies = [ "indexmap", "serde_core", @@ -4068,27 +4134,27 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.7.5+spec-1.1.0" +version = "1.1.0+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347" +checksum = "97251a7c317e03ad83774a8752a7e81fb6067740609f75ea2b585b569a59198f" dependencies = [ "serde_core", ] [[package]] name = "toml_parser" -version = "1.0.6+spec-1.1.0" +version = "1.1.0+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3198b4b0a8e11f09dd03e133c0280504d0801269e9afa46362ffde1cbeebf44" +checksum = "2334f11ee363607eb04df9b8fc8a13ca1715a72ba8662a26ac285c98aabb4011" dependencies = [ "winnow", ] [[package]] name = "toml_writer" -version = "1.0.6+spec-1.1.0" +version = "1.1.0+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab16f14aed21ee8bfd8ec22513f7287cd4a91aa92e44edfe2c17ddd004e92607" +checksum = "d282ade6016312faf3e41e57ebbba0c073e4056dab1232ab1cb624199648f8ed" [[package]] name = "twox-hash" @@ -4138,15 +4204,6 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80d7ff825a6a654ee85a63e80f92f054f904f21e7d12da4e22f9834a4aaa35bc" -[[package]] -name = "unic-normal" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f09d64d33589a94628bc2aeb037f35c2e25f3f049c7348b5aa5580b48e6bba62" -dependencies = [ - "unic-ucd-normal", -] - [[package]] name = "unic-ucd-age" version = "0.9.0" @@ -4158,61 +4215,6 @@ dependencies = [ "unic-ucd-version", ] -[[package]] -name = "unic-ucd-bidi" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1d568b51222484e1f8209ce48caa6b430bf352962b877d592c29ab31fb53d8c" -dependencies = [ - "unic-char-property", - "unic-char-range", - "unic-ucd-version", -] - -[[package]] -name = "unic-ucd-category" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b8d4591f5fcfe1bd4453baaf803c40e1b1e69ff8455c47620440b46efef91c0" -dependencies = [ - "matches", - "unic-char-property", - "unic-char-range", - "unic-ucd-version", -] - -[[package]] -name = "unic-ucd-hangul" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb1dc690e19010e1523edb9713224cba5ef55b54894fe33424439ec9a40c0054" -dependencies = [ - "unic-ucd-version", -] - -[[package]] -name = "unic-ucd-ident" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e230a37c0381caa9219d67cf063aa3a375ffed5bf541a452db16e744bdab6987" -dependencies = [ - "unic-char-property", - "unic-char-range", - "unic-ucd-version", -] - -[[package]] -name = "unic-ucd-normal" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86aed873b8202d22b13859dda5fe7c001d271412c31d411fd9b827e030569410" -dependencies = [ - "unic-char-property", - "unic-char-range", - "unic-ucd-hangul", - "unic-ucd-version", -] - [[package]] name = "unic-ucd-version" version = "0.9.0" @@ -4222,12 +4224,6 @@ dependencies = [ "unic-common", ] -[[package]] -name = "unicode-bidi-mirroring" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dfa6e8c60bb66d49db113e0125ee8711b7647b5579dc7f5f19c42357ed039fe" - [[package]] name = "unicode-casing" version = "0.1.1" @@ -4303,12 +4299,6 @@ dependencies = [ "rand 0.8.5", ] -[[package]] -name = "unindent" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" - [[package]] name = "untrusted" version = "0.7.1" @@ -4321,6 +4311,18 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "utf16_iter" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "utf8parse" version = "0.2.2" @@ -4329,9 +4331,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.20.0" +version = "1.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee48d38b119b0cd71fe4141b30f5ba9c7c5d9f4e7a3a8b4a674e4b6ef789976f" +checksum = "a68d3c8f01c0cfa54a75291d83601161799e4a89a39e0929f4b0354d88757a37" dependencies = [ "atomic", "js-sys", @@ -4435,59 +4437,27 @@ dependencies = [ ] [[package]] -name = "wasmparser" -version = "0.243.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6d8db401b0528ec316dfbe579e6ab4152d61739cfe076706d2009127970159d" -dependencies = [ - "bitflags 2.10.0", - "hashbrown 0.15.5", - "indexmap", - "serde", -] - -[[package]] -name = "wasmtime-environ" -version = "41.0.1" +name = "wasmtime-internal-core" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37b9af430b11ff3cd63fbef54cf38e26154089c179316b8a5e400b8ba2d0ebf1" +checksum = "e671917bb6856ae360cb59d7aaf26f1cfd042c7b924319dd06fd380739fc0b2e" dependencies = [ - "anyhow", - "cranelift-bitset", - "cranelift-entity", - "gimli", - "indexmap", - "log", - "object", - "postcard", - "serde", - "serde_derive", - "smallvec", - "target-lexicon", - "wasmparser", + "hashbrown 0.16.1", + "libm", ] [[package]] name = "wasmtime-internal-jit-icache-coherence" -version = "41.0.1" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85b46da671c07242b5f5eab491b12d6c25dd26929f1693c055fcca94489ef8f5" +checksum = "9b3112806515fac8495883885eb8dbdde849988ae91fe6beb544c0d7c0f4c9aa" dependencies = [ "cfg-if", "libc", - "wasmtime-environ", + "wasmtime-internal-core", "windows-sys 0.61.2", ] -[[package]] -name = "wasmtime-internal-math" -version = "41.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d1f0763c6f6f78e410f964db9f53d9b84ab4cc336945e81f0b78717b0a9934e" -dependencies = [ - "libm", -] - [[package]] name = "web-sys" version = "0.3.85" @@ -4509,29 +4479,27 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12bed680863276c63889429bfd6cab3b99943659923822de1c8a39c49e4d722c" +checksum = "22cfaf3c063993ff62e73cb4311efde4db1efb31ab78a3e5c457939ad5cc0bed" dependencies = [ "rustls-pki-types", ] [[package]] name = "which" -version = "8.0.0" +version = "8.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3fabb953106c3c8eea8306e4393700d7657561cb43122571b172bbfb7c7ba1d" +checksum = "81995fafaaaf6ae47a7d0cc83c67caf92aeb7e5331650ae6ff856f7c0c60c459" dependencies = [ - "env_home", - "rustix", - "winsafe", + "libc", ] [[package]] name = "wide" -version = "0.8.3" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13ca908d26e4786149c48efcf6c0ea09ab0e06d1fe3c17dc1b4b0f1ca4a7e788" +checksum = "ac11b009ebeae802ed758530b6496784ebfee7a87b9abfbcaf3bbe25b814eb25" dependencies = [ "bytemuck", "safe_arch", @@ -4866,32 +4834,38 @@ checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" [[package]] name = "winnow" -version = "0.7.14" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a5364e9d77fcdeeaa6062ced926ee3381faa2ee02d3eb83a5c27a8825540829" +checksum = "a90e88e4667264a994d34e6d1ab2d26d398dcdca8b7f52bec8668957517fc7d8" [[package]] name = "winresource" -version = "0.1.30" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e287ced0f21cd11f4035fe946fd3af145f068d1acb708afd248100f89ec7432d" +checksum = "0986a8b1d586b7d3e4fe3d9ea39fb451ae22869dcea4aa109d287a374d866087" dependencies = [ "toml", "version_check", ] -[[package]] -name = "winsafe" -version = "0.0.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d135d17ab770252ad95e9a872d365cf3090e3be864a34ab46f48555993efc904" - [[package]] name = "wit-bindgen" version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + +[[package]] +name = "writeable" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" + [[package]] name = "x509-cert" version = "0.2.5" @@ -4908,9 +4882,9 @@ dependencies = [ [[package]] name = "x509-parser" -version = "0.18.0" +version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb3e137310115a65136898d2079f003ce33331a6c4b0d51f1531d1be082b6425" +checksum = "d43b0f71ce057da06bc0851b23ee24f3f86190b07203dd8f567d0b706a185202" dependencies = [ "asn1-rs", "data-encoding", @@ -4930,12 +4904,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8aa498d22c9bbaf482329839bc5620c46be275a19a812e9a22a2b07529a642a" [[package]] -name = "xz2" -version = "0.1.7" +name = "yoke" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" +checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" dependencies = [ - "lzma-sys", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", ] [[package]] @@ -4958,6 +4946,27 @@ dependencies = [ "syn", ] +[[package]] +name = "zerofrom" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + [[package]] name = "zeroize" version = "1.8.2" @@ -4978,12 +4987,51 @@ dependencies = [ "syn", ] +[[package]] +name = "zerotrie" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "zlib-rs" version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40990edd51aae2c2b6907af74ffb635029d5788228222c4bb811e9351c0caad3" +[[package]] +name = "zlib-rs" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7948af682ccbc3342b6e9420e8c51c1fe5d7bf7756002b4a3c6cabfe96a7e3c" + [[package]] name = "zmij" version = "1.0.17" diff --git a/Cargo.toml b/Cargo.toml index 54d1fdda41f..7bd8b8f3374 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,8 @@ repository.workspace = true license.workspace = true [features] -default = ["threading", "stdlib", "stdio", "importlib", "ssl-rustls"] +default = ["threading", "stdlib", "stdio", "importlib", "ssl-rustls", "host_env"] +host_env = ["rustpython-vm/host_env", "rustpython-stdlib?/host_env"] importlib = ["rustpython-vm/importlib"] encodings = ["rustpython-vm/encodings"] stdio = ["rustpython-vm/stdio"] @@ -33,8 +34,7 @@ winresource = "0.1" rustpython-compiler = { workspace = true } rustpython-pylib = { workspace = true, optional = true } rustpython-stdlib = { workspace = true, optional = true, features = ["compiler"] } -rustpython-vm = { workspace = true, features = ["compiler"] } -ruff_python_parser = { workspace = true } +rustpython-vm = { workspace = true, features = ["compiler", "gc"] } cfg-if = { workspace = true } log = { workspace = true } @@ -53,8 +53,9 @@ rustyline = { workspace = true } [dev-dependencies] criterion = { workspace = true } -pyo3 = { version = "0.27", features = ["auto-initialize"] } +pyo3 = { version = "0.28.2", features = ["auto-initialize"] } rustpython-stdlib = { workspace = true } +ruff_python_parser = { workspace = true } [[bench]] name = "execution" @@ -76,10 +77,21 @@ opt-level = 3 # https://github.com/rust-lang/rust/issues/92869 # lto = "thin" -# Doesn't change often +# Some crates don't change as much but benefit more from +# more expensive optimization passes, so we selectively +# decrease codegen-units in some cases. [profile.release.package.rustpython-doc] codegen-units = 1 +[profile.release.package.rustpython-literal] +codegen-units = 1 + +[profile.release.package.rustpython-common] +codegen-units = 1 + +[profile.release.package.rustpython-wtf8] +codegen-units = 1 + [profile.bench] lto = "thin" codegen-units = 1 @@ -89,6 +101,7 @@ opt-level = 3 lto = "thin" [patch.crates-io] +parking_lot_core = { git = "https://github.com/youknowone/parking_lot", branch = "rustpython" } # REDOX START, Uncomment when you want to compile/check with redoxer # REDOX END @@ -120,7 +133,7 @@ members = [ exclude = ["pymath"] [workspace.package] -version = "0.4.0" +version = "0.5.0" authors = ["RustPython Team"] edition = "2024" rust-version = "1.93.0" @@ -128,35 +141,43 @@ repository = "https://github.com/RustPython/RustPython" license = "MIT" [workspace.dependencies] -rustpython-compiler-core = { path = "crates/compiler-core", version = "0.4.0" } -rustpython-compiler = { path = "crates/compiler", version = "0.4.0" } -rustpython-codegen = { path = "crates/codegen", version = "0.4.0" } -rustpython-common = { path = "crates/common", version = "0.4.0" } -rustpython-derive = { path = "crates/derive", version = "0.4.0" } -rustpython-derive-impl = { path = "crates/derive-impl", version = "0.4.0" } -rustpython-jit = { path = "crates/jit", version = "0.4.0" } -rustpython-literal = { path = "crates/literal", version = "0.4.0" } -rustpython-vm = { path = "crates/vm", default-features = false, version = "0.4.0" } -rustpython-pylib = { path = "crates/pylib", version = "0.4.0" } -rustpython-stdlib = { path = "crates/stdlib", default-features = false, version = "0.4.0" } -rustpython-sre_engine = { path = "crates/sre_engine", version = "0.4.0" } -rustpython-wtf8 = { path = "crates/wtf8", version = "0.4.0" } -rustpython-doc = { path = "crates/doc", version = "0.4.0" } - -# Ruff tag 0.14.14 is based on commit 8b2e7b36f246b990fe473a84eef25ff429e59ecf +rustpython-compiler-core = { path = "crates/compiler-core", version = "0.5.0" } +rustpython-compiler = { path = "crates/compiler", version = "0.5.0" } +rustpython-codegen = { path = "crates/codegen", version = "0.5.0" } +rustpython-common = { path = "crates/common", version = "0.5.0" } +rustpython-derive = { path = "crates/derive", version = "0.5.0" } +rustpython-derive-impl = { path = "crates/derive-impl", version = "0.5.0" } +rustpython-jit = { path = "crates/jit", version = "0.5.0" } +rustpython-literal = { path = "crates/literal", version = "0.5.0" } +rustpython-vm = { path = "crates/vm", default-features = false, version = "0.5.0" } +rustpython-pylib = { path = "crates/pylib", version = "0.5.0" } +rustpython-stdlib = { path = "crates/stdlib", default-features = false, version = "0.5.0" } +rustpython-sre_engine = { path = "crates/sre_engine", version = "0.5.0" } +rustpython-wtf8 = { path = "crates/wtf8", version = "0.5.0" } +rustpython-doc = { path = "crates/doc", version = "0.5.0" } + +# Use RustPython-packaged Ruff crates from the published fork while keeping +# existing crate names in the codebase. +ruff_python_parser = { package = "rustpython-ruff_python_parser", version = "0.15.8" } +ruff_python_ast = { package = "rustpython-ruff_python_ast", version = "0.15.8" } +ruff_text_size = { package = "rustpython-ruff_text_size", version = "0.15.8" } +ruff_source_file = { package = "rustpython-ruff_source_file", version = "0.15.8" } +# To update ruff crates, comment out the above lines and uncomment the following lines to pull directly from the Ruff repository at the specified commit hash. +# Ruff tag 0.15.8 is based on commit c2a8815842f9dc5d24ec19385eae0f1a7188b0d9 # at the time of this capture. We use the commit hash to ensure reproducible builds. -ruff_python_parser = { git = "https://github.com/astral-sh/ruff.git", rev = "8b2e7b36f246b990fe473a84eef25ff429e59ecf" } -ruff_python_ast = { git = "https://github.com/astral-sh/ruff.git", rev = "8b2e7b36f246b990fe473a84eef25ff429e59ecf" } -ruff_text_size = { git = "https://github.com/astral-sh/ruff.git", rev = "8b2e7b36f246b990fe473a84eef25ff429e59ecf" } -ruff_source_file = { git = "https://github.com/astral-sh/ruff.git", rev = "8b2e7b36f246b990fe473a84eef25ff429e59ecf" } +# ruff_python_parser = { git = "https://github.com/astral-sh/ruff.git", rev = "c2a8815842f9dc5d24ec19385eae0f1a7188b0d9" } +# ruff_python_ast = { git = "https://github.com/astral-sh/ruff.git", rev = "c2a8815842f9dc5d24ec19385eae0f1a7188b0d9" } +# ruff_text_size = { git = "https://github.com/astral-sh/ruff.git", rev = "c2a8815842f9dc5d24ec19385eae0f1a7188b0d9" } +# ruff_source_file = { git = "https://github.com/astral-sh/ruff.git", rev = "c2a8815842f9dc5d24ec19385eae0f1a7188b0d9" } phf = { version = "0.13.1", default-features = false, features = ["macros"]} ahash = "0.8.12" ascii = "1.1" -bitflags = "2.9.4" +bitflags = "2.11.0" +bitflagset = "0.0.3" bstr = "1" cfg-if = "1.0" -chrono = { version = "0.4.43", default-features = false, features = ["clock", "oldtime", "std"] } +chrono = { version = "0.4.44", default-features = false, features = ["clock", "oldtime", "std"] } constant_time_eq = "0.4" criterion = { version = "0.8", features = ["html_reports"] } crossbeam-utils = "0.8.21" @@ -168,50 +189,44 @@ indexmap = { version = "2.13.0", features = ["std"] } insta = "1.46" itertools = "0.14.0" is-macro = "0.3.7" -junction = "1.4.1" -libc = "0.2.180" +junction = "1.4.2" +libc = "0.2.183" libffi = "5" log = "0.4.29" nix = { version = "0.30", features = ["fs", "user", "process", "term", "time", "signal", "ioctl", "socket", "sched", "zerocopy", "dir", "hostname", "net", "poll"] } -malachite-bigint = "0.9" -malachite-q = "0.9" -malachite-base = "0.9" -memchr = "2.7.4" +malachite-bigint = "0.9.1" +malachite-q = "0.9.1" +malachite-base = "0.9.1" +memchr = "2.8.0" num-complex = "0.4.6" num-integer = "0.1.46" num-traits = "0.2" num_enum = { version = "0.7", default-features = false } optional = "0.5" -once_cell = "1.20.3" parking_lot = "0.12.3" paste = "1.0.15" proc-macro2 = "1.0.105" -pymath = { version = "0.1.5", features = ["mul_add", "malachite-bigint", "complex"] } -quote = "1.0.44" +pymath = { version = "0.2.0", features = ["mul_add", "malachite-bigint", "complex"] } +quote = "1.0.45" radium = "1.1.1" rand = "0.9" rand_core = { version = "0.9", features = ["os_rng"] } rustix = { version = "1.1", features = ["event"] } rustyline = "17.0.1" serde = { package = "serde_core", version = "1.0.225", default-features = false, features = ["alloc"] } -schannel = "0.1.28" +schannel = "0.1.29" scoped-tls = "1" scopeguard = "1" static_assertions = "1.1" -strum = "0.27" -strum_macros = "0.27" +strum = "0.28" +strum_macros = "0.28" syn = "2" thiserror = "2.0" -thread_local = "1.1.9" +icu_properties = "2" +icu_normalizer = "2" unicode-casing = "0.1.1" -unic-char-property = "0.9.0" -unic-normal = "0.9.0" unic-ucd-age = "0.9.0" -unic-ucd-bidi = "0.9.0" -unic-ucd-category = "0.9.0" -unic-ucd-ident = "0.9.0" unicode_names2 = "2.0.0" -unicode-bidi-mirroring = "0.4" widestring = "1.2.0" windows-sys = "0.61.2" wasm-bindgen = "0.2.106" @@ -224,9 +239,9 @@ unsafe_op_in_unsafe_fn = "deny" elided_lifetimes_in_paths = "warn" [workspace.lints.clippy] -# alloc_instead_of_core = "warn" -# std_instead_of_alloc = "warn" -# std_instead_of_core = "warn" +alloc_instead_of_core = "warn" +std_instead_of_alloc = "warn" +std_instead_of_core = "warn" perf = "warn" style = "warn" complexity = "warn" diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 24c149eebef..7573f0f2640 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -65,7 +65,7 @@ $ pytest -v Rust unit tests can be run with `cargo`: ```shell -$ cargo test --workspace --exclude rustpython_wasm +$ cargo test --workspace --exclude rustpython_wasm --exclude rustpython-venvlauncher ``` Python unit tests can be run by compiling RustPython and running the test module: diff --git a/Lib/_ast_unparse.py b/Lib/_ast_unparse.py new file mode 100644 index 00000000000..1c8741b5a55 --- /dev/null +++ b/Lib/_ast_unparse.py @@ -0,0 +1,1161 @@ +# This module contains ``ast.unparse()``, defined here +# to improve the import time for the ``ast`` module. +import sys +from _ast import * +from ast import NodeVisitor +from contextlib import contextmanager, nullcontext +from enum import IntEnum, auto, _simple_enum + +# Large float and imaginary literals get turned into infinities in the AST. +# We unparse those infinities to INFSTR. +_INFSTR = "1e" + repr(sys.float_info.max_10_exp + 1) + +@_simple_enum(IntEnum) +class _Precedence: + """Precedence table that originated from python grammar.""" + + NAMED_EXPR = auto() # := + TUPLE = auto() # , + YIELD = auto() # 'yield', 'yield from' + TEST = auto() # 'if'-'else', 'lambda' + OR = auto() # 'or' + AND = auto() # 'and' + NOT = auto() # 'not' + CMP = auto() # '<', '>', '==', '>=', '<=', '!=', + # 'in', 'not in', 'is', 'is not' + EXPR = auto() + BOR = EXPR # '|' + BXOR = auto() # '^' + BAND = auto() # '&' + SHIFT = auto() # '<<', '>>' + ARITH = auto() # '+', '-' + TERM = auto() # '*', '@', '/', '%', '//' + FACTOR = auto() # unary '+', '-', '~' + POWER = auto() # '**' + AWAIT = auto() # 'await' + ATOM = auto() + + def next(self): + try: + return self.__class__(self + 1) + except ValueError: + return self + + +_SINGLE_QUOTES = ("'", '"') +_MULTI_QUOTES = ('"""', "'''") +_ALL_QUOTES = (*_SINGLE_QUOTES, *_MULTI_QUOTES) + +class Unparser(NodeVisitor): + """Methods in this class recursively traverse an AST and + output source code for the abstract syntax; original formatting + is disregarded.""" + + def __init__(self): + self._source = [] + self._precedences = {} + self._type_ignores = {} + self._indent = 0 + self._in_try_star = False + self._in_interactive = False + + def interleave(self, inter, f, seq): + """Call f on each item in seq, calling inter() in between.""" + seq = iter(seq) + try: + f(next(seq)) + except StopIteration: + pass + else: + for x in seq: + inter() + f(x) + + def items_view(self, traverser, items): + """Traverse and separate the given *items* with a comma and append it to + the buffer. If *items* is a single item sequence, a trailing comma + will be added.""" + if len(items) == 1: + traverser(items[0]) + self.write(",") + else: + self.interleave(lambda: self.write(", "), traverser, items) + + def maybe_newline(self): + """Adds a newline if it isn't the start of generated source""" + if self._source: + self.write("\n") + + def maybe_semicolon(self): + """Adds a "; " delimiter if it isn't the start of generated source""" + if self._source: + self.write("; ") + + def fill(self, text="", *, allow_semicolon=True): + """Indent a piece of text and append it, according to the current + indentation level, or only delineate with semicolon if applicable""" + if self._in_interactive and not self._indent and allow_semicolon: + self.maybe_semicolon() + self.write(text) + else: + self.maybe_newline() + self.write(" " * self._indent + text) + + def write(self, *text): + """Add new source parts""" + self._source.extend(text) + + @contextmanager + def buffered(self, buffer = None): + if buffer is None: + buffer = [] + + original_source = self._source + self._source = buffer + yield buffer + self._source = original_source + + @contextmanager + def block(self, *, extra = None): + """A context manager for preparing the source for blocks. It adds + the character':', increases the indentation on enter and decreases + the indentation on exit. If *extra* is given, it will be directly + appended after the colon character. + """ + self.write(":") + if extra: + self.write(extra) + self._indent += 1 + yield + self._indent -= 1 + + @contextmanager + def delimit(self, start, end): + """A context manager for preparing the source for expressions. It adds + *start* to the buffer and enters, after exit it adds *end*.""" + + self.write(start) + yield + self.write(end) + + def delimit_if(self, start, end, condition): + if condition: + return self.delimit(start, end) + else: + return nullcontext() + + def require_parens(self, precedence, node): + """Shortcut to adding precedence related parens""" + return self.delimit_if("(", ")", self.get_precedence(node) > precedence) + + def get_precedence(self, node): + return self._precedences.get(node, _Precedence.TEST) + + def set_precedence(self, precedence, *nodes): + for node in nodes: + self._precedences[node] = precedence + + def get_raw_docstring(self, node): + """If a docstring node is found in the body of the *node* parameter, + return that docstring node, None otherwise. + + Logic mirrored from ``_PyAST_GetDocString``.""" + if not isinstance( + node, (AsyncFunctionDef, FunctionDef, ClassDef, Module) + ) or len(node.body) < 1: + return None + node = node.body[0] + if not isinstance(node, Expr): + return None + node = node.value + if isinstance(node, Constant) and isinstance(node.value, str): + return node + + def get_type_comment(self, node): + comment = self._type_ignores.get(node.lineno) or node.type_comment + if comment is not None: + return f" # type: {comment}" + + def traverse(self, node): + if isinstance(node, list): + for item in node: + self.traverse(item) + else: + super().visit(node) + + # Note: as visit() resets the output text, do NOT rely on + # NodeVisitor.generic_visit to handle any nodes (as it calls back in to + # the subclass visit() method, which resets self._source to an empty list) + def visit(self, node): + """Outputs a source code string that, if converted back to an ast + (using ast.parse) will generate an AST equivalent to *node*""" + self._source = [] + self.traverse(node) + return "".join(self._source) + + def _write_docstring_and_traverse_body(self, node): + if (docstring := self.get_raw_docstring(node)): + self._write_docstring(docstring) + self.traverse(node.body[1:]) + else: + self.traverse(node.body) + + def visit_Module(self, node): + self._type_ignores = { + ignore.lineno: f"ignore{ignore.tag}" + for ignore in node.type_ignores + } + try: + self._write_docstring_and_traverse_body(node) + finally: + self._type_ignores.clear() + + def visit_Interactive(self, node): + self._in_interactive = True + try: + self._write_docstring_and_traverse_body(node) + finally: + self._in_interactive = False + + def visit_FunctionType(self, node): + with self.delimit("(", ")"): + self.interleave( + lambda: self.write(", "), self.traverse, node.argtypes + ) + + self.write(" -> ") + self.traverse(node.returns) + + def visit_Expr(self, node): + self.fill() + self.set_precedence(_Precedence.YIELD, node.value) + self.traverse(node.value) + + def visit_NamedExpr(self, node): + with self.require_parens(_Precedence.NAMED_EXPR, node): + self.set_precedence(_Precedence.ATOM, node.target, node.value) + self.traverse(node.target) + self.write(" := ") + self.traverse(node.value) + + def visit_Import(self, node): + self.fill("import ") + self.interleave(lambda: self.write(", "), self.traverse, node.names) + + def visit_ImportFrom(self, node): + self.fill("from ") + self.write("." * (node.level or 0)) + if node.module: + self.write(node.module) + self.write(" import ") + self.interleave(lambda: self.write(", "), self.traverse, node.names) + + def visit_Assign(self, node): + self.fill() + for target in node.targets: + self.set_precedence(_Precedence.TUPLE, target) + self.traverse(target) + self.write(" = ") + self.traverse(node.value) + if type_comment := self.get_type_comment(node): + self.write(type_comment) + + def visit_AugAssign(self, node): + self.fill() + self.traverse(node.target) + self.write(" " + self.binop[node.op.__class__.__name__] + "= ") + self.traverse(node.value) + + def visit_AnnAssign(self, node): + self.fill() + with self.delimit_if("(", ")", not node.simple and isinstance(node.target, Name)): + self.traverse(node.target) + self.write(": ") + self.traverse(node.annotation) + if node.value: + self.write(" = ") + self.traverse(node.value) + + def visit_Return(self, node): + self.fill("return") + if node.value: + self.write(" ") + self.traverse(node.value) + + def visit_Pass(self, node): + self.fill("pass") + + def visit_Break(self, node): + self.fill("break") + + def visit_Continue(self, node): + self.fill("continue") + + def visit_Delete(self, node): + self.fill("del ") + self.interleave(lambda: self.write(", "), self.traverse, node.targets) + + def visit_Assert(self, node): + self.fill("assert ") + self.traverse(node.test) + if node.msg: + self.write(", ") + self.traverse(node.msg) + + def visit_Global(self, node): + self.fill("global ") + self.interleave(lambda: self.write(", "), self.write, node.names) + + def visit_Nonlocal(self, node): + self.fill("nonlocal ") + self.interleave(lambda: self.write(", "), self.write, node.names) + + def visit_Await(self, node): + with self.require_parens(_Precedence.AWAIT, node): + self.write("await") + if node.value: + self.write(" ") + self.set_precedence(_Precedence.ATOM, node.value) + self.traverse(node.value) + + def visit_Yield(self, node): + with self.require_parens(_Precedence.YIELD, node): + self.write("yield") + if node.value: + self.write(" ") + self.set_precedence(_Precedence.ATOM, node.value) + self.traverse(node.value) + + def visit_YieldFrom(self, node): + with self.require_parens(_Precedence.YIELD, node): + self.write("yield from ") + if not node.value: + raise ValueError("Node can't be used without a value attribute.") + self.set_precedence(_Precedence.ATOM, node.value) + self.traverse(node.value) + + def visit_Raise(self, node): + self.fill("raise") + if not node.exc: + if node.cause: + raise ValueError(f"Node can't use cause without an exception.") + return + self.write(" ") + self.traverse(node.exc) + if node.cause: + self.write(" from ") + self.traverse(node.cause) + + def do_visit_try(self, node): + self.fill("try", allow_semicolon=False) + with self.block(): + self.traverse(node.body) + for ex in node.handlers: + self.traverse(ex) + if node.orelse: + self.fill("else", allow_semicolon=False) + with self.block(): + self.traverse(node.orelse) + if node.finalbody: + self.fill("finally", allow_semicolon=False) + with self.block(): + self.traverse(node.finalbody) + + def visit_Try(self, node): + prev_in_try_star = self._in_try_star + try: + self._in_try_star = False + self.do_visit_try(node) + finally: + self._in_try_star = prev_in_try_star + + def visit_TryStar(self, node): + prev_in_try_star = self._in_try_star + try: + self._in_try_star = True + self.do_visit_try(node) + finally: + self._in_try_star = prev_in_try_star + + def visit_ExceptHandler(self, node): + self.fill("except*" if self._in_try_star else "except", allow_semicolon=False) + if node.type: + self.write(" ") + self.traverse(node.type) + if node.name: + self.write(" as ") + self.write(node.name) + with self.block(): + self.traverse(node.body) + + def visit_ClassDef(self, node): + self.maybe_newline() + for deco in node.decorator_list: + self.fill("@", allow_semicolon=False) + self.traverse(deco) + self.fill("class " + node.name, allow_semicolon=False) + if hasattr(node, "type_params"): + self._type_params_helper(node.type_params) + with self.delimit_if("(", ")", condition = node.bases or node.keywords): + comma = False + for e in node.bases: + if comma: + self.write(", ") + else: + comma = True + self.traverse(e) + for e in node.keywords: + if comma: + self.write(", ") + else: + comma = True + self.traverse(e) + + with self.block(): + self._write_docstring_and_traverse_body(node) + + def visit_FunctionDef(self, node): + self._function_helper(node, "def") + + def visit_AsyncFunctionDef(self, node): + self._function_helper(node, "async def") + + def _function_helper(self, node, fill_suffix): + self.maybe_newline() + for deco in node.decorator_list: + self.fill("@", allow_semicolon=False) + self.traverse(deco) + def_str = fill_suffix + " " + node.name + self.fill(def_str, allow_semicolon=False) + if hasattr(node, "type_params"): + self._type_params_helper(node.type_params) + with self.delimit("(", ")"): + self.traverse(node.args) + if node.returns: + self.write(" -> ") + self.traverse(node.returns) + with self.block(extra=self.get_type_comment(node)): + self._write_docstring_and_traverse_body(node) + + def _type_params_helper(self, type_params): + if type_params is not None and len(type_params) > 0: + with self.delimit("[", "]"): + self.interleave(lambda: self.write(", "), self.traverse, type_params) + + def visit_TypeVar(self, node): + self.write(node.name) + if node.bound: + self.write(": ") + self.traverse(node.bound) + if node.default_value: + self.write(" = ") + self.traverse(node.default_value) + + def visit_TypeVarTuple(self, node): + self.write("*" + node.name) + if node.default_value: + self.write(" = ") + self.traverse(node.default_value) + + def visit_ParamSpec(self, node): + self.write("**" + node.name) + if node.default_value: + self.write(" = ") + self.traverse(node.default_value) + + def visit_TypeAlias(self, node): + self.fill("type ") + self.traverse(node.name) + self._type_params_helper(node.type_params) + self.write(" = ") + self.traverse(node.value) + + def visit_For(self, node): + self._for_helper("for ", node) + + def visit_AsyncFor(self, node): + self._for_helper("async for ", node) + + def _for_helper(self, fill, node): + self.fill(fill, allow_semicolon=False) + self.set_precedence(_Precedence.TUPLE, node.target) + self.traverse(node.target) + self.write(" in ") + self.traverse(node.iter) + with self.block(extra=self.get_type_comment(node)): + self.traverse(node.body) + if node.orelse: + self.fill("else", allow_semicolon=False) + with self.block(): + self.traverse(node.orelse) + + def visit_If(self, node): + self.fill("if ", allow_semicolon=False) + self.traverse(node.test) + with self.block(): + self.traverse(node.body) + # collapse nested ifs into equivalent elifs. + while node.orelse and len(node.orelse) == 1 and isinstance(node.orelse[0], If): + node = node.orelse[0] + self.fill("elif ", allow_semicolon=False) + self.traverse(node.test) + with self.block(): + self.traverse(node.body) + # final else + if node.orelse: + self.fill("else", allow_semicolon=False) + with self.block(): + self.traverse(node.orelse) + + def visit_While(self, node): + self.fill("while ", allow_semicolon=False) + self.traverse(node.test) + with self.block(): + self.traverse(node.body) + if node.orelse: + self.fill("else", allow_semicolon=False) + with self.block(): + self.traverse(node.orelse) + + def visit_With(self, node): + self.fill("with ", allow_semicolon=False) + self.interleave(lambda: self.write(", "), self.traverse, node.items) + with self.block(extra=self.get_type_comment(node)): + self.traverse(node.body) + + def visit_AsyncWith(self, node): + self.fill("async with ", allow_semicolon=False) + self.interleave(lambda: self.write(", "), self.traverse, node.items) + with self.block(extra=self.get_type_comment(node)): + self.traverse(node.body) + + def _str_literal_helper( + self, string, *, quote_types=_ALL_QUOTES, escape_special_whitespace=False + ): + """Helper for writing string literals, minimizing escapes. + Returns the tuple (string literal to write, possible quote types). + """ + def escape_char(c): + # \n and \t are non-printable, but we only escape them if + # escape_special_whitespace is True + if not escape_special_whitespace and c in "\n\t": + return c + # Always escape backslashes and other non-printable characters + if c == "\\" or not c.isprintable(): + return c.encode("unicode_escape").decode("ascii") + return c + + escaped_string = "".join(map(escape_char, string)) + possible_quotes = quote_types + if "\n" in escaped_string: + possible_quotes = [q for q in possible_quotes if q in _MULTI_QUOTES] + possible_quotes = [q for q in possible_quotes if q not in escaped_string] + if not possible_quotes: + # If there aren't any possible_quotes, fallback to using repr + # on the original string. Try to use a quote from quote_types, + # e.g., so that we use triple quotes for docstrings. + string = repr(string) + quote = next((q for q in quote_types if string[0] in q), string[0]) + return string[1:-1], [quote] + if escaped_string: + # Sort so that we prefer '''"''' over """\"""" + possible_quotes.sort(key=lambda q: q[0] == escaped_string[-1]) + # If we're using triple quotes and we'd need to escape a final + # quote, escape it + if possible_quotes[0][0] == escaped_string[-1]: + assert len(possible_quotes[0]) == 3 + escaped_string = escaped_string[:-1] + "\\" + escaped_string[-1] + return escaped_string, possible_quotes + + def _write_str_avoiding_backslashes(self, string, *, quote_types=_ALL_QUOTES): + """Write string literal value with a best effort attempt to avoid backslashes.""" + string, quote_types = self._str_literal_helper(string, quote_types=quote_types) + quote_type = quote_types[0] + self.write(f"{quote_type}{string}{quote_type}") + + def _ftstring_helper(self, parts): + new_parts = [] + quote_types = list(_ALL_QUOTES) + fallback_to_repr = False + for value, is_constant in parts: + if is_constant: + value, new_quote_types = self._str_literal_helper( + value, + quote_types=quote_types, + escape_special_whitespace=True, + ) + if set(new_quote_types).isdisjoint(quote_types): + fallback_to_repr = True + break + quote_types = new_quote_types + else: + if "\n" in value: + quote_types = [q for q in quote_types if q in _MULTI_QUOTES] + assert quote_types + + new_quote_types = [q for q in quote_types if q not in value] + if new_quote_types: + quote_types = new_quote_types + new_parts.append(value) + + if fallback_to_repr: + # If we weren't able to find a quote type that works for all parts + # of the JoinedStr, fallback to using repr and triple single quotes. + quote_types = ["'''"] + new_parts.clear() + for value, is_constant in parts: + if is_constant: + value = repr('"' + value) # force repr to use single quotes + expected_prefix = "'\"" + assert value.startswith(expected_prefix), repr(value) + value = value[len(expected_prefix):-1] + new_parts.append(value) + + value = "".join(new_parts) + quote_type = quote_types[0] + self.write(f"{quote_type}{value}{quote_type}") + + def _write_ftstring(self, values, prefix): + self.write(prefix) + fstring_parts = [] + for value in values: + with self.buffered() as buffer: + self._write_ftstring_inner(value) + fstring_parts.append( + ("".join(buffer), isinstance(value, Constant)) + ) + self._ftstring_helper(fstring_parts) + + def visit_JoinedStr(self, node): + self._write_ftstring(node.values, "f") + + def visit_TemplateStr(self, node): + self._write_ftstring(node.values, "t") + + def _write_ftstring_inner(self, node, is_format_spec=False): + if isinstance(node, JoinedStr): + # for both the f-string itself, and format_spec + for value in node.values: + self._write_ftstring_inner(value, is_format_spec=is_format_spec) + elif isinstance(node, Constant) and isinstance(node.value, str): + value = node.value.replace("{", "{{").replace("}", "}}") + + if is_format_spec: + value = value.replace("\\", "\\\\") + value = value.replace("'", "\\'") + value = value.replace('"', '\\"') + value = value.replace("\n", "\\n") + self.write(value) + elif isinstance(node, FormattedValue): + self.visit_FormattedValue(node) + elif isinstance(node, Interpolation): + self.visit_Interpolation(node) + else: + raise ValueError(f"Unexpected node inside JoinedStr, {node!r}") + + def _unparse_interpolation_value(self, inner): + unparser = type(self)() + unparser.set_precedence(_Precedence.TEST.next(), inner) + return unparser.visit(inner) + + def _write_interpolation(self, node, use_str_attr=False): + with self.delimit("{", "}"): + if use_str_attr: + expr = node.str + else: + expr = self._unparse_interpolation_value(node.value) + if expr.startswith("{"): + # Separate pair of opening brackets as "{ {" + self.write(" ") + self.write(expr) + if node.conversion != -1: + self.write(f"!{chr(node.conversion)}") + if node.format_spec: + self.write(":") + self._write_ftstring_inner(node.format_spec, is_format_spec=True) + + def visit_FormattedValue(self, node): + self._write_interpolation(node) + + def visit_Interpolation(self, node): + # If `str` is set to `None`, use the `value` to generate the source code. + self._write_interpolation(node, use_str_attr=node.str is not None) + + def visit_Name(self, node): + self.write(node.id) + + def _write_docstring(self, node): + self.fill(allow_semicolon=False) + if node.kind == "u": + self.write("u") + self._write_str_avoiding_backslashes(node.value, quote_types=_MULTI_QUOTES) + + def _write_constant(self, value): + if isinstance(value, (float, complex)): + # Substitute overflowing decimal literal for AST infinities, + # and inf - inf for NaNs. + self.write( + repr(value) + .replace("inf", _INFSTR) + .replace("nan", f"({_INFSTR}-{_INFSTR})") + ) + else: + self.write(repr(value)) + + def visit_Constant(self, node): + value = node.value + if isinstance(value, tuple): + with self.delimit("(", ")"): + self.items_view(self._write_constant, value) + elif value is ...: + self.write("...") + else: + if node.kind == "u": + self.write("u") + self._write_constant(node.value) + + def visit_List(self, node): + with self.delimit("[", "]"): + self.interleave(lambda: self.write(", "), self.traverse, node.elts) + + def visit_ListComp(self, node): + with self.delimit("[", "]"): + self.traverse(node.elt) + for gen in node.generators: + self.traverse(gen) + + def visit_GeneratorExp(self, node): + with self.delimit("(", ")"): + self.traverse(node.elt) + for gen in node.generators: + self.traverse(gen) + + def visit_SetComp(self, node): + with self.delimit("{", "}"): + self.traverse(node.elt) + for gen in node.generators: + self.traverse(gen) + + def visit_DictComp(self, node): + with self.delimit("{", "}"): + self.traverse(node.key) + self.write(": ") + self.traverse(node.value) + for gen in node.generators: + self.traverse(gen) + + def visit_comprehension(self, node): + if node.is_async: + self.write(" async for ") + else: + self.write(" for ") + self.set_precedence(_Precedence.TUPLE, node.target) + self.traverse(node.target) + self.write(" in ") + self.set_precedence(_Precedence.TEST.next(), node.iter, *node.ifs) + self.traverse(node.iter) + for if_clause in node.ifs: + self.write(" if ") + self.traverse(if_clause) + + def visit_IfExp(self, node): + with self.require_parens(_Precedence.TEST, node): + self.set_precedence(_Precedence.TEST.next(), node.body, node.test) + self.traverse(node.body) + self.write(" if ") + self.traverse(node.test) + self.write(" else ") + self.set_precedence(_Precedence.TEST, node.orelse) + self.traverse(node.orelse) + + def visit_Set(self, node): + if node.elts: + with self.delimit("{", "}"): + self.interleave(lambda: self.write(", "), self.traverse, node.elts) + else: + # `{}` would be interpreted as a dictionary literal, and + # `set` might be shadowed. Thus: + self.write('{*()}') + + def visit_Dict(self, node): + def write_key_value_pair(k, v): + self.traverse(k) + self.write(": ") + self.traverse(v) + + def write_item(item): + k, v = item + if k is None: + # for dictionary unpacking operator in dicts {**{'y': 2}} + # see PEP 448 for details + self.write("**") + self.set_precedence(_Precedence.EXPR, v) + self.traverse(v) + else: + write_key_value_pair(k, v) + + with self.delimit("{", "}"): + self.interleave( + lambda: self.write(", "), write_item, zip(node.keys, node.values) + ) + + def visit_Tuple(self, node): + with self.delimit_if( + "(", + ")", + len(node.elts) == 0 or self.get_precedence(node) > _Precedence.TUPLE + ): + self.items_view(self.traverse, node.elts) + + unop = {"Invert": "~", "Not": "not", "UAdd": "+", "USub": "-"} + unop_precedence = { + "not": _Precedence.NOT, + "~": _Precedence.FACTOR, + "+": _Precedence.FACTOR, + "-": _Precedence.FACTOR, + } + + def visit_UnaryOp(self, node): + operator = self.unop[node.op.__class__.__name__] + operator_precedence = self.unop_precedence[operator] + with self.require_parens(operator_precedence, node): + self.write(operator) + # factor prefixes (+, -, ~) shouldn't be separated + # from the value they belong, (e.g: +1 instead of + 1) + if operator_precedence is not _Precedence.FACTOR: + self.write(" ") + self.set_precedence(operator_precedence, node.operand) + self.traverse(node.operand) + + binop = { + "Add": "+", + "Sub": "-", + "Mult": "*", + "MatMult": "@", + "Div": "/", + "Mod": "%", + "LShift": "<<", + "RShift": ">>", + "BitOr": "|", + "BitXor": "^", + "BitAnd": "&", + "FloorDiv": "//", + "Pow": "**", + } + + binop_precedence = { + "+": _Precedence.ARITH, + "-": _Precedence.ARITH, + "*": _Precedence.TERM, + "@": _Precedence.TERM, + "/": _Precedence.TERM, + "%": _Precedence.TERM, + "<<": _Precedence.SHIFT, + ">>": _Precedence.SHIFT, + "|": _Precedence.BOR, + "^": _Precedence.BXOR, + "&": _Precedence.BAND, + "//": _Precedence.TERM, + "**": _Precedence.POWER, + } + + binop_rassoc = frozenset(("**",)) + def visit_BinOp(self, node): + operator = self.binop[node.op.__class__.__name__] + operator_precedence = self.binop_precedence[operator] + with self.require_parens(operator_precedence, node): + if operator in self.binop_rassoc: + left_precedence = operator_precedence.next() + right_precedence = operator_precedence + else: + left_precedence = operator_precedence + right_precedence = operator_precedence.next() + + self.set_precedence(left_precedence, node.left) + self.traverse(node.left) + self.write(f" {operator} ") + self.set_precedence(right_precedence, node.right) + self.traverse(node.right) + + cmpops = { + "Eq": "==", + "NotEq": "!=", + "Lt": "<", + "LtE": "<=", + "Gt": ">", + "GtE": ">=", + "Is": "is", + "IsNot": "is not", + "In": "in", + "NotIn": "not in", + } + + def visit_Compare(self, node): + with self.require_parens(_Precedence.CMP, node): + self.set_precedence(_Precedence.CMP.next(), node.left, *node.comparators) + self.traverse(node.left) + for o, e in zip(node.ops, node.comparators): + self.write(" " + self.cmpops[o.__class__.__name__] + " ") + self.traverse(e) + + boolops = {"And": "and", "Or": "or"} + boolop_precedence = {"and": _Precedence.AND, "or": _Precedence.OR} + + def visit_BoolOp(self, node): + operator = self.boolops[node.op.__class__.__name__] + operator_precedence = self.boolop_precedence[operator] + + def increasing_level_traverse(node): + nonlocal operator_precedence + operator_precedence = operator_precedence.next() + self.set_precedence(operator_precedence, node) + self.traverse(node) + + with self.require_parens(operator_precedence, node): + s = f" {operator} " + self.interleave(lambda: self.write(s), increasing_level_traverse, node.values) + + def visit_Attribute(self, node): + self.set_precedence(_Precedence.ATOM, node.value) + self.traverse(node.value) + # Special case: 3.__abs__() is a syntax error, so if node.value + # is an integer literal then we need to either parenthesize + # it or add an extra space to get 3 .__abs__(). + if isinstance(node.value, Constant) and isinstance(node.value.value, int): + self.write(" ") + self.write(".") + self.write(node.attr) + + def visit_Call(self, node): + self.set_precedence(_Precedence.ATOM, node.func) + self.traverse(node.func) + with self.delimit("(", ")"): + comma = False + for e in node.args: + if comma: + self.write(", ") + else: + comma = True + self.traverse(e) + for e in node.keywords: + if comma: + self.write(", ") + else: + comma = True + self.traverse(e) + + def visit_Subscript(self, node): + def is_non_empty_tuple(slice_value): + return ( + isinstance(slice_value, Tuple) + and slice_value.elts + ) + + self.set_precedence(_Precedence.ATOM, node.value) + self.traverse(node.value) + with self.delimit("[", "]"): + if is_non_empty_tuple(node.slice): + # parentheses can be omitted if the tuple isn't empty + self.items_view(self.traverse, node.slice.elts) + else: + self.traverse(node.slice) + + def visit_Starred(self, node): + self.write("*") + self.set_precedence(_Precedence.EXPR, node.value) + self.traverse(node.value) + + def visit_Ellipsis(self, node): + self.write("...") + + def visit_Slice(self, node): + if node.lower: + self.traverse(node.lower) + self.write(":") + if node.upper: + self.traverse(node.upper) + if node.step: + self.write(":") + self.traverse(node.step) + + def visit_Match(self, node): + self.fill("match ", allow_semicolon=False) + self.traverse(node.subject) + with self.block(): + for case in node.cases: + self.traverse(case) + + def visit_arg(self, node): + self.write(node.arg) + if node.annotation: + self.write(": ") + self.traverse(node.annotation) + + def visit_arguments(self, node): + first = True + # normal arguments + all_args = node.posonlyargs + node.args + defaults = [None] * (len(all_args) - len(node.defaults)) + node.defaults + for index, elements in enumerate(zip(all_args, defaults), 1): + a, d = elements + if first: + first = False + else: + self.write(", ") + self.traverse(a) + if d: + self.write("=") + self.traverse(d) + if index == len(node.posonlyargs): + self.write(", /") + + # varargs, or bare '*' if no varargs but keyword-only arguments present + if node.vararg or node.kwonlyargs: + if first: + first = False + else: + self.write(", ") + self.write("*") + if node.vararg: + self.write(node.vararg.arg) + if node.vararg.annotation: + self.write(": ") + self.traverse(node.vararg.annotation) + + # keyword-only arguments + if node.kwonlyargs: + for a, d in zip(node.kwonlyargs, node.kw_defaults): + self.write(", ") + self.traverse(a) + if d: + self.write("=") + self.traverse(d) + + # kwargs + if node.kwarg: + if first: + first = False + else: + self.write(", ") + self.write("**" + node.kwarg.arg) + if node.kwarg.annotation: + self.write(": ") + self.traverse(node.kwarg.annotation) + + def visit_keyword(self, node): + if node.arg is None: + self.write("**") + else: + self.write(node.arg) + self.write("=") + self.traverse(node.value) + + def visit_Lambda(self, node): + with self.require_parens(_Precedence.TEST, node): + self.write("lambda") + with self.buffered() as buffer: + self.traverse(node.args) + if buffer: + self.write(" ", *buffer) + self.write(": ") + self.set_precedence(_Precedence.TEST, node.body) + self.traverse(node.body) + + def visit_alias(self, node): + self.write(node.name) + if node.asname: + self.write(" as " + node.asname) + + def visit_withitem(self, node): + self.traverse(node.context_expr) + if node.optional_vars: + self.write(" as ") + self.traverse(node.optional_vars) + + def visit_match_case(self, node): + self.fill("case ", allow_semicolon=False) + self.traverse(node.pattern) + if node.guard: + self.write(" if ") + self.traverse(node.guard) + with self.block(): + self.traverse(node.body) + + def visit_MatchValue(self, node): + self.traverse(node.value) + + def visit_MatchSingleton(self, node): + self._write_constant(node.value) + + def visit_MatchSequence(self, node): + with self.delimit("[", "]"): + self.interleave( + lambda: self.write(", "), self.traverse, node.patterns + ) + + def visit_MatchStar(self, node): + name = node.name + if name is None: + name = "_" + self.write(f"*{name}") + + def visit_MatchMapping(self, node): + def write_key_pattern_pair(pair): + k, p = pair + self.traverse(k) + self.write(": ") + self.traverse(p) + + with self.delimit("{", "}"): + keys = node.keys + self.interleave( + lambda: self.write(", "), + write_key_pattern_pair, + zip(keys, node.patterns, strict=True), + ) + rest = node.rest + if rest is not None: + if keys: + self.write(", ") + self.write(f"**{rest}") + + def visit_MatchClass(self, node): + self.set_precedence(_Precedence.ATOM, node.cls) + self.traverse(node.cls) + with self.delimit("(", ")"): + patterns = node.patterns + self.interleave( + lambda: self.write(", "), self.traverse, patterns + ) + attrs = node.kwd_attrs + if attrs: + def write_attr_pattern(pair): + attr, pattern = pair + self.write(f"{attr}=") + self.traverse(pattern) + + if patterns: + self.write(", ") + self.interleave( + lambda: self.write(", "), + write_attr_pattern, + zip(attrs, node.kwd_patterns, strict=True), + ) + + def visit_MatchAs(self, node): + name = node.name + pattern = node.pattern + if name is None: + self.write("_") + elif pattern is None: + self.write(node.name) + else: + with self.require_parens(_Precedence.TEST, node): + self.set_precedence(_Precedence.BOR, node.pattern) + self.traverse(node.pattern) + self.write(f" as {node.name}") + + def visit_MatchOr(self, node): + with self.require_parens(_Precedence.BOR, node): + self.set_precedence(_Precedence.BOR.next(), *node.patterns) + self.interleave(lambda: self.write(" | "), self.traverse, node.patterns) diff --git a/Lib/_compat_pickle.py b/Lib/_compat_pickle.py index 17b9010278f..60793c391ae 100644 --- a/Lib/_compat_pickle.py +++ b/Lib/_compat_pickle.py @@ -22,7 +22,6 @@ 'tkMessageBox': 'tkinter.messagebox', 'ScrolledText': 'tkinter.scrolledtext', 'Tkconstants': 'tkinter.constants', - 'Tix': 'tkinter.tix', 'ttk': 'tkinter.ttk', 'Tkinter': 'tkinter', 'markupbase': '_markupbase', @@ -257,3 +256,4 @@ for excname in PYTHON3_IMPORTERROR_EXCEPTIONS: REVERSE_NAME_MAPPING[('builtins', excname)] = ('exceptions', 'ImportError') +del excname diff --git a/Lib/_markupbase.py b/Lib/_markupbase.py index 3ad7e279960..614f0cd16dd 100644 --- a/Lib/_markupbase.py +++ b/Lib/_markupbase.py @@ -13,7 +13,7 @@ _markedsectionclose = re.compile(r']\s*]\s*>') # An analysis of the MS-Word extensions is available at -# http://www.planetpublish.com/xmlarena/xap/Thursday/WordtoXML.pdf +# http://web.archive.org/web/20060321153828/http://www.planetpublish.com/xmlarena/xap/Thursday/WordtoXML.pdf _msmarkedsectionclose = re.compile(r']\s*>') diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index bb55ee423cf..4da6e507736 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -1,16 +1,223 @@ # This file is generated by scripts/generate_opcode_metadata.py -# for RustPython bytecode format (CPython 3.13 compatible opcode numbers). +# for RustPython bytecode format (CPython 3.14 compatible opcode numbers). # Do not edit! -_specializations = {} +_specializations = { + "RESUME": [ + "RESUME_CHECK", + ], + "LOAD_CONST": [ + "LOAD_CONST_MORTAL", + "LOAD_CONST_IMMORTAL", + ], + "TO_BOOL": [ + "TO_BOOL_ALWAYS_TRUE", + "TO_BOOL_BOOL", + "TO_BOOL_INT", + "TO_BOOL_LIST", + "TO_BOOL_NONE", + "TO_BOOL_STR", + ], + "BINARY_OP": [ + "BINARY_OP_MULTIPLY_INT", + "BINARY_OP_ADD_INT", + "BINARY_OP_SUBTRACT_INT", + "BINARY_OP_MULTIPLY_FLOAT", + "BINARY_OP_ADD_FLOAT", + "BINARY_OP_SUBTRACT_FLOAT", + "BINARY_OP_ADD_UNICODE", + "BINARY_OP_SUBSCR_LIST_INT", + "BINARY_OP_SUBSCR_LIST_SLICE", + "BINARY_OP_SUBSCR_TUPLE_INT", + "BINARY_OP_SUBSCR_STR_INT", + "BINARY_OP_SUBSCR_DICT", + "BINARY_OP_SUBSCR_GETITEM", + "BINARY_OP_EXTEND", + "BINARY_OP_INPLACE_ADD_UNICODE", + ], + "STORE_SUBSCR": [ + "STORE_SUBSCR_DICT", + "STORE_SUBSCR_LIST_INT", + ], + "SEND": [ + "SEND_GEN", + ], + "UNPACK_SEQUENCE": [ + "UNPACK_SEQUENCE_TWO_TUPLE", + "UNPACK_SEQUENCE_TUPLE", + "UNPACK_SEQUENCE_LIST", + ], + "STORE_ATTR": [ + "STORE_ATTR_INSTANCE_VALUE", + "STORE_ATTR_SLOT", + "STORE_ATTR_WITH_HINT", + ], + "LOAD_GLOBAL": [ + "LOAD_GLOBAL_MODULE", + "LOAD_GLOBAL_BUILTIN", + ], + "LOAD_SUPER_ATTR": [ + "LOAD_SUPER_ATTR_ATTR", + "LOAD_SUPER_ATTR_METHOD", + ], + "LOAD_ATTR": [ + "LOAD_ATTR_INSTANCE_VALUE", + "LOAD_ATTR_MODULE", + "LOAD_ATTR_WITH_HINT", + "LOAD_ATTR_SLOT", + "LOAD_ATTR_CLASS", + "LOAD_ATTR_CLASS_WITH_METACLASS_CHECK", + "LOAD_ATTR_PROPERTY", + "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", + "LOAD_ATTR_METHOD_WITH_VALUES", + "LOAD_ATTR_METHOD_NO_DICT", + "LOAD_ATTR_METHOD_LAZY_DICT", + "LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", + "LOAD_ATTR_NONDESCRIPTOR_NO_DICT", + ], + "COMPARE_OP": [ + "COMPARE_OP_FLOAT", + "COMPARE_OP_INT", + "COMPARE_OP_STR", + ], + "CONTAINS_OP": [ + "CONTAINS_OP_SET", + "CONTAINS_OP_DICT", + ], + "JUMP_BACKWARD": [ + "JUMP_BACKWARD_NO_JIT", + "JUMP_BACKWARD_JIT", + ], + "FOR_ITER": [ + "FOR_ITER_LIST", + "FOR_ITER_TUPLE", + "FOR_ITER_RANGE", + "FOR_ITER_GEN", + ], + "CALL": [ + "CALL_BOUND_METHOD_EXACT_ARGS", + "CALL_PY_EXACT_ARGS", + "CALL_TYPE_1", + "CALL_STR_1", + "CALL_TUPLE_1", + "CALL_BUILTIN_CLASS", + "CALL_BUILTIN_O", + "CALL_BUILTIN_FAST", + "CALL_BUILTIN_FAST_WITH_KEYWORDS", + "CALL_LEN", + "CALL_ISINSTANCE", + "CALL_LIST_APPEND", + "CALL_METHOD_DESCRIPTOR_O", + "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", + "CALL_METHOD_DESCRIPTOR_NOARGS", + "CALL_METHOD_DESCRIPTOR_FAST", + "CALL_ALLOC_AND_ENTER_INIT", + "CALL_PY_GENERAL", + "CALL_BOUND_METHOD_GENERAL", + "CALL_NON_PY_GENERAL", + ], + "CALL_KW": [ + "CALL_KW_BOUND_METHOD", + "CALL_KW_PY", + "CALL_KW_NON_PY", + ], +} -_specialized_opmap = {} +_specialized_opmap = { + 'BINARY_OP_ADD_FLOAT': 129, + 'BINARY_OP_ADD_INT': 130, + 'BINARY_OP_ADD_UNICODE': 131, + 'BINARY_OP_EXTEND': 132, + 'BINARY_OP_INPLACE_ADD_UNICODE': 3, + 'BINARY_OP_MULTIPLY_FLOAT': 133, + 'BINARY_OP_MULTIPLY_INT': 134, + 'BINARY_OP_SUBSCR_DICT': 135, + 'BINARY_OP_SUBSCR_GETITEM': 136, + 'BINARY_OP_SUBSCR_LIST_INT': 137, + 'BINARY_OP_SUBSCR_LIST_SLICE': 138, + 'BINARY_OP_SUBSCR_STR_INT': 139, + 'BINARY_OP_SUBSCR_TUPLE_INT': 140, + 'BINARY_OP_SUBTRACT_FLOAT': 141, + 'BINARY_OP_SUBTRACT_INT': 142, + 'CALL_ALLOC_AND_ENTER_INIT': 143, + 'CALL_BOUND_METHOD_EXACT_ARGS': 144, + 'CALL_BOUND_METHOD_GENERAL': 145, + 'CALL_BUILTIN_CLASS': 146, + 'CALL_BUILTIN_FAST': 147, + 'CALL_BUILTIN_FAST_WITH_KEYWORDS': 148, + 'CALL_BUILTIN_O': 149, + 'CALL_ISINSTANCE': 150, + 'CALL_KW_BOUND_METHOD': 151, + 'CALL_KW_NON_PY': 152, + 'CALL_KW_PY': 153, + 'CALL_LEN': 154, + 'CALL_LIST_APPEND': 155, + 'CALL_METHOD_DESCRIPTOR_FAST': 156, + 'CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS': 157, + 'CALL_METHOD_DESCRIPTOR_NOARGS': 158, + 'CALL_METHOD_DESCRIPTOR_O': 159, + 'CALL_NON_PY_GENERAL': 160, + 'CALL_PY_EXACT_ARGS': 161, + 'CALL_PY_GENERAL': 162, + 'CALL_STR_1': 163, + 'CALL_TUPLE_1': 164, + 'CALL_TYPE_1': 165, + 'COMPARE_OP_FLOAT': 166, + 'COMPARE_OP_INT': 167, + 'COMPARE_OP_STR': 168, + 'CONTAINS_OP_DICT': 169, + 'CONTAINS_OP_SET': 170, + 'FOR_ITER_GEN': 171, + 'FOR_ITER_LIST': 172, + 'FOR_ITER_RANGE': 173, + 'FOR_ITER_TUPLE': 174, + 'JUMP_BACKWARD_JIT': 175, + 'JUMP_BACKWARD_NO_JIT': 176, + 'LOAD_ATTR_CLASS': 177, + 'LOAD_ATTR_CLASS_WITH_METACLASS_CHECK': 178, + 'LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN': 179, + 'LOAD_ATTR_INSTANCE_VALUE': 180, + 'LOAD_ATTR_METHOD_LAZY_DICT': 181, + 'LOAD_ATTR_METHOD_NO_DICT': 182, + 'LOAD_ATTR_METHOD_WITH_VALUES': 183, + 'LOAD_ATTR_MODULE': 184, + 'LOAD_ATTR_NONDESCRIPTOR_NO_DICT': 185, + 'LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES': 186, + 'LOAD_ATTR_PROPERTY': 187, + 'LOAD_ATTR_SLOT': 188, + 'LOAD_ATTR_WITH_HINT': 189, + 'LOAD_CONST_IMMORTAL': 190, + 'LOAD_CONST_MORTAL': 191, + 'LOAD_GLOBAL_BUILTIN': 192, + 'LOAD_GLOBAL_MODULE': 193, + 'LOAD_SUPER_ATTR_ATTR': 194, + 'LOAD_SUPER_ATTR_METHOD': 195, + 'RESUME_CHECK': 196, + 'SEND_GEN': 197, + 'STORE_ATTR_INSTANCE_VALUE': 198, + 'STORE_ATTR_SLOT': 199, + 'STORE_ATTR_WITH_HINT': 200, + 'STORE_SUBSCR_DICT': 201, + 'STORE_SUBSCR_LIST_INT': 202, + 'TO_BOOL_ALWAYS_TRUE': 203, + 'TO_BOOL_BOOL': 204, + 'TO_BOOL_INT': 205, + 'TO_BOOL_LIST': 206, + 'TO_BOOL_NONE': 207, + 'TO_BOOL_STR': 208, + 'UNPACK_SEQUENCE_LIST': 209, + 'UNPACK_SEQUENCE_TUPLE': 210, + 'UNPACK_SEQUENCE_TWO_TUPLE': 211, +} opmap = { 'CACHE': 0, + 'RESERVED': 17, + 'RESUME': 128, + 'INSTRUMENTED_LINE': 254, + 'ENTER_EXECUTOR': 255, 'BINARY_SLICE': 1, 'BUILD_TEMPLATE': 2, - 'BINARY_OP_INPLACE_ADD_UNICODE': 3, 'CALL_FUNCTION_EX': 4, 'CHECK_EG_MATCH': 5, 'CHECK_EXC_MATCH': 6, @@ -24,7 +231,6 @@ 'GET_AITER': 14, 'GET_ANEXT': 15, 'GET_ITER': 16, - 'RESERVED': 17, 'GET_LEN': 18, 'GET_YIELD_FROM_ITER': 19, 'INTERPRETER_EXIT': 20, @@ -128,90 +334,6 @@ 'UNPACK_EX': 118, 'UNPACK_SEQUENCE': 119, 'YIELD_VALUE': 120, - 'RESUME': 128, - 'BINARY_OP_ADD_FLOAT': 129, - 'BINARY_OP_ADD_INT': 130, - 'BINARY_OP_ADD_UNICODE': 131, - 'BINARY_OP_EXTEND': 132, - 'BINARY_OP_MULTIPLY_FLOAT': 133, - 'BINARY_OP_MULTIPLY_INT': 134, - 'BINARY_OP_SUBSCR_DICT': 135, - 'BINARY_OP_SUBSCR_GETITEM': 136, - 'BINARY_OP_SUBSCR_LIST_INT': 137, - 'BINARY_OP_SUBSCR_LIST_SLICE': 138, - 'BINARY_OP_SUBSCR_STR_INT': 139, - 'BINARY_OP_SUBSCR_TUPLE_INT': 140, - 'BINARY_OP_SUBTRACT_FLOAT': 141, - 'BINARY_OP_SUBTRACT_INT': 142, - 'CALL_ALLOC_AND_ENTER_INIT': 143, - 'CALL_BOUND_METHOD_EXACT_ARGS': 144, - 'CALL_BOUND_METHOD_GENERAL': 145, - 'CALL_BUILTIN_CLASS': 146, - 'CALL_BUILTIN_FAST': 147, - 'CALL_BUILTIN_FAST_WITH_KEYWORDS': 148, - 'CALL_BUILTIN_O': 149, - 'CALL_ISINSTANCE': 150, - 'CALL_KW_BOUND_METHOD': 151, - 'CALL_KW_NON_PY': 152, - 'CALL_KW_PY': 153, - 'CALL_LEN': 154, - 'CALL_LIST_APPEND': 155, - 'CALL_METHOD_DESCRIPTOR_FAST': 156, - 'CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS': 157, - 'CALL_METHOD_DESCRIPTOR_NOARGS': 158, - 'CALL_METHOD_DESCRIPTOR_O': 159, - 'CALL_NON_PY_GENERAL': 160, - 'CALL_PY_EXACT_ARGS': 161, - 'CALL_PY_GENERAL': 162, - 'CALL_STR_1': 163, - 'CALL_TUPLE_1': 164, - 'CALL_TYPE_1': 165, - 'COMPARE_OP_FLOAT': 166, - 'COMPARE_OP_INT': 167, - 'COMPARE_OP_STR': 168, - 'CONTAINS_OP_DICT': 169, - 'CONTAINS_OP_SET': 170, - 'FOR_ITER_GEN': 171, - 'FOR_ITER_LIST': 172, - 'FOR_ITER_RANGE': 173, - 'FOR_ITER_TUPLE': 174, - 'JUMP_BACKWARD_JIT': 175, - 'JUMP_BACKWARD_NO_JIT': 176, - 'LOAD_ATTR_CLASS': 177, - 'LOAD_ATTR_CLASS_WITH_METACLASS_CHECK': 178, - 'LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN': 179, - 'LOAD_ATTR_INSTANCE_VALUE': 180, - 'LOAD_ATTR_METHOD_LAZY_DICT': 181, - 'LOAD_ATTR_METHOD_NO_DICT': 182, - 'LOAD_ATTR_METHOD_WITH_VALUES': 183, - 'LOAD_ATTR_MODULE': 184, - 'LOAD_ATTR_NONDESCRIPTOR_NO_DICT': 185, - 'LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES': 186, - 'LOAD_ATTR_PROPERTY': 187, - 'LOAD_ATTR_SLOT': 188, - 'LOAD_ATTR_WITH_HINT': 189, - 'LOAD_CONST_IMMORTAL': 190, - 'LOAD_CONST_MORTAL': 191, - 'LOAD_GLOBAL_BUILTIN': 192, - 'LOAD_GLOBAL_MODULE': 193, - 'LOAD_SUPER_ATTR_ATTR': 194, - 'LOAD_SUPER_ATTR_METHOD': 195, - 'RESUME_CHECK': 196, - 'SEND_GEN': 197, - 'STORE_ATTR_INSTANCE_VALUE': 198, - 'STORE_ATTR_SLOT': 199, - 'STORE_ATTR_WITH_HINT': 200, - 'STORE_SUBSCR_DICT': 201, - 'STORE_SUBSCR_LIST_INT': 202, - 'TO_BOOL_ALWAYS_TRUE': 203, - 'TO_BOOL_BOOL': 204, - 'TO_BOOL_INT': 205, - 'TO_BOOL_LIST': 206, - 'TO_BOOL_NONE': 207, - 'TO_BOOL_STR': 208, - 'UNPACK_SEQUENCE_LIST': 209, - 'UNPACK_SEQUENCE_TUPLE': 210, - 'UNPACK_SEQUENCE_TWO_TUPLE': 211, 'INSTRUMENTED_END_FOR': 234, 'INSTRUMENTED_POP_ITER': 235, 'INSTRUMENTED_END_SEND': 236, @@ -232,8 +354,6 @@ 'INSTRUMENTED_CALL_KW': 251, 'INSTRUMENTED_CALL_FUNCTION_EX': 252, 'INSTRUMENTED_JUMP_BACKWARD': 253, - 'INSTRUMENTED_LINE': 254, - 'ENTER_EXECUTOR': 255, 'ANNOTATIONS_PLACEHOLDER': 256, 'JUMP': 257, 'JUMP_IF_FALSE': 258, @@ -247,6 +367,5 @@ 'STORE_FAST_MAYBE_NULL': 266, } -# CPython 3.13 compatible: opcodes < 44 have no argument -HAVE_ARGUMENT = 44 -MIN_INSTRUMENTED_OPCODE = 236 +HAVE_ARGUMENT = 43 +MIN_INSTRUMENTED_OPCODE = 234 diff --git a/Lib/_pycodecs.py b/Lib/_pycodecs.py index d0efa9ad6bb..98dec3c614d 100644 --- a/Lib/_pycodecs.py +++ b/Lib/_pycodecs.py @@ -22,10 +22,10 @@ The builtin Unicode codecs use the following interface: - _encode(Unicode_object[,errors='strict']) -> + _encode(Unicode_object[,errors='strict']) -> (string object, bytes consumed) - _decode(char_buffer_obj[,errors='strict']) -> + _decode(char_buffer_obj[,errors='strict']) -> (Unicode object, bytes consumed) _encode() interfaces also accept non-Unicode object as @@ -44,47 +44,76 @@ From PyPy v1.0.0 """ -#from unicodecodec import * - -__all__ = ['register', 'lookup', 'lookup_error', 'register_error', 'encode', 'decode', - 'latin_1_encode', 'mbcs_decode', 'readbuffer_encode', 'escape_encode', - 'utf_8_decode', 'raw_unicode_escape_decode', 'utf_7_decode', - 'unicode_escape_encode', 'latin_1_decode', 'utf_16_decode', - 'unicode_escape_decode', 'ascii_decode', 'charmap_encode', 'charmap_build', - 'unicode_internal_encode', 'unicode_internal_decode', 'utf_16_ex_decode', - 'escape_decode', 'charmap_decode', 'utf_7_encode', 'mbcs_encode', - 'ascii_encode', 'utf_16_encode', 'raw_unicode_escape_encode', 'utf_8_encode', - 'utf_16_le_encode', 'utf_16_be_encode', 'utf_16_le_decode', 'utf_16_be_decode',] +# from unicodecodec import * + +__all__ = [ + "register", + "lookup", + "lookup_error", + "register_error", + "encode", + "decode", + "latin_1_encode", + "mbcs_decode", + "readbuffer_encode", + "escape_encode", + "utf_8_decode", + "raw_unicode_escape_decode", + "utf_7_decode", + "unicode_escape_encode", + "latin_1_decode", + "utf_16_decode", + "unicode_escape_decode", + "ascii_decode", + "charmap_encode", + "charmap_build", + "unicode_internal_encode", + "unicode_internal_decode", + "utf_16_ex_decode", + "escape_decode", + "charmap_decode", + "utf_7_encode", + "mbcs_encode", + "ascii_encode", + "utf_16_encode", + "raw_unicode_escape_encode", + "utf_8_encode", + "utf_16_le_encode", + "utf_16_be_encode", + "utf_16_le_decode", + "utf_16_be_decode", + "utf_32_ex_decode", +] import sys import warnings from _codecs import * -def latin_1_encode( obj, errors='strict'): - """None - """ +def latin_1_encode(obj, errors="strict"): + """None""" res = PyUnicode_EncodeLatin1(obj, len(obj), errors) res = bytes(res) return res, len(obj) + + # XXX MBCS codec might involve ctypes ? def mbcs_decode(): - """None - """ + """None""" pass -def readbuffer_encode( obj, errors='strict'): - """None - """ + +def readbuffer_encode(obj, errors="strict"): + """None""" if isinstance(obj, str): res = obj.encode() else: res = bytes(obj) return res, len(obj) -def escape_encode( obj, errors='strict'): - """None - """ + +def escape_encode(obj, errors="strict"): + """None""" if not isinstance(obj, bytes): raise TypeError("must be bytes") s = repr(obj).encode() @@ -93,85 +122,88 @@ def escape_encode( obj, errors='strict'): v = v.replace(b"'", b"\\'").replace(b'\\"', b'"') return v, len(obj) -def raw_unicode_escape_decode( data, errors='strict', final=False): - """None - """ - res = PyUnicode_DecodeRawUnicodeEscape(data, len(data), errors, final) - res = ''.join(res) - return res, len(data) -def utf_7_decode( data, errors='strict'): - """None - """ - res = PyUnicode_DecodeUTF7(data, len(data), errors) - res = ''.join(res) - return res, len(data) +def raw_unicode_escape_decode(data, errors="strict", final=True): + """None""" + res, consumed = PyUnicode_DecodeRawUnicodeEscape(data, len(data), errors, final) + res = "".join(res) + return res, consumed -def unicode_escape_encode( obj, errors='strict'): - """None - """ + +def utf_7_decode(data, errors="strict", final=False): + """None""" + res, consumed = PyUnicode_DecodeUTF7(data, len(data), errors, final) + res = "".join(res) + return res, consumed + + +def unicode_escape_encode(obj, errors="strict"): + """None""" res = unicodeescape_string(obj, len(obj), 0) - res = b''.join(res) + res = b"".join(res) return res, len(obj) -def latin_1_decode( data, errors='strict'): - """None - """ + +def latin_1_decode(data, errors="strict"): + """None""" res = PyUnicode_DecodeLatin1(data, len(data), errors) - res = ''.join(res) + res = "".join(res) return res, len(data) -def utf_16_decode( data, errors='strict', final=False): - """None - """ + +def utf_16_decode(data, errors="strict", final=False): + """None""" consumed = len(data) if final: consumed = 0 - res, consumed, byteorder = PyUnicode_DecodeUTF16Stateful(data, len(data), errors, 'native', final) - res = ''.join(res) + res, consumed, byteorder = PyUnicode_DecodeUTF16Stateful( + data, len(data), errors, "native", final + ) + res = "".join(res) return res, consumed -def unicode_escape_decode( data, errors='strict', final=False): - """None - """ - res = PyUnicode_DecodeUnicodeEscape(data, len(data), errors, final) - res = ''.join(res) - return res, len(data) +def unicode_escape_decode(data, errors="strict", final=True): + """None""" + res, consumed = PyUnicode_DecodeUnicodeEscape(data, len(data), errors, final) + res = "".join(res) + return res, consumed -def ascii_decode( data, errors='strict'): - """None - """ + +def ascii_decode(data, errors="strict"): + """None""" res = PyUnicode_DecodeASCII(data, len(data), errors) - res = ''.join(res) + res = "".join(res) return res, len(data) -def charmap_encode(obj, errors='strict', mapping='latin-1'): - """None - """ + +def charmap_encode(obj, errors="strict", mapping="latin-1"): + """None""" res = PyUnicode_EncodeCharmap(obj, len(obj), mapping, errors) res = bytes(res) return res, len(obj) + def charmap_build(s): return {ord(c): i for i, c in enumerate(s)} + if sys.maxunicode == 65535: unicode_bytes = 2 else: unicode_bytes = 4 -def unicode_internal_encode( obj, errors='strict'): - """None - """ + +def unicode_internal_encode(obj, errors="strict"): + """None""" if type(obj) == str: p = bytearray() t = [ord(x) for x in obj] for i in t: b = bytearray() for j in range(unicode_bytes): - b.append(i%256) + b.append(i % 256) i >>= 8 if sys.byteorder == "big": b.reverse() @@ -179,12 +211,12 @@ def unicode_internal_encode( obj, errors='strict'): res = bytes(p) return res, len(res) else: - res = "You can do better than this" # XXX make this right + res = "You can do better than this" # XXX make this right return res, len(res) -def unicode_internal_decode( unistr, errors='strict'): - """None - """ + +def unicode_internal_decode(unistr, errors="strict"): + """None""" if type(unistr) == str: return unistr, len(unistr) else: @@ -198,165 +230,418 @@ def unicode_internal_decode( unistr, errors='strict'): start = 0 stop = unicode_bytes step = 1 - while i < len(unistr)-unicode_bytes+1: + while i < len(unistr) - unicode_bytes + 1: t = 0 h = 0 for j in range(start, stop, step): - t += ord(unistr[i+j])<<(h*8) + t += ord(unistr[i + j]) << (h * 8) h += 1 i += unicode_bytes p += chr(t) - res = ''.join(p) + res = "".join(p) return res, len(res) -def utf_16_ex_decode( data, errors='strict', byteorder=0, final=0): - """None - """ + +def utf_16_ex_decode(data, errors="strict", byteorder=0, final=0): + """None""" if byteorder == 0: - bm = 'native' + bm = "native" elif byteorder == -1: - bm = 'little' + bm = "little" else: - bm = 'big' + bm = "big" consumed = len(data) if final: consumed = 0 - res, consumed, byteorder = PyUnicode_DecodeUTF16Stateful(data, len(data), errors, bm, final) - res = ''.join(res) + res, consumed, byteorder = PyUnicode_DecodeUTF16Stateful( + data, len(data), errors, bm, final + ) + res = "".join(res) return res, consumed, byteorder -# XXX needs error messages when the input is invalid -def escape_decode(data, errors='strict'): - """None - """ + +def utf_32_ex_decode(data, errors="strict", byteorder=0, final=0): + """None""" + if byteorder == 0: + if len(data) < 4: + if final and len(data): + if sys.byteorder == "little": + bm = "little" + else: + bm = "big" + res, consumed, _ = PyUnicode_DecodeUTF32Stateful( + data, len(data), errors, bm, final + ) + return "".join(res), consumed, 0 + return "", 0, 0 + if data[0:4] == b"\xff\xfe\x00\x00": + res, consumed, _ = PyUnicode_DecodeUTF32Stateful( + data[4:], len(data) - 4, errors, "little", final + ) + return "".join(res), consumed + 4, -1 + if data[0:4] == b"\x00\x00\xfe\xff": + res, consumed, _ = PyUnicode_DecodeUTF32Stateful( + data[4:], len(data) - 4, errors, "big", final + ) + return "".join(res), consumed + 4, 1 + if sys.byteorder == "little": + bm = "little" + else: + bm = "big" + res, consumed, _ = PyUnicode_DecodeUTF32Stateful( + data, len(data), errors, bm, final + ) + return "".join(res), consumed, 0 + + if byteorder == -1: + res, consumed, _ = PyUnicode_DecodeUTF32Stateful( + data, len(data), errors, "little", final + ) + return "".join(res), consumed, -1 + + res, consumed, _ = PyUnicode_DecodeUTF32Stateful( + data, len(data), errors, "big", final + ) + return "".join(res), consumed, 1 + + +def _is_hex_digit(b): + return ( + 0x30 <= b <= 0x39 # 0-9 + or 0x41 <= b <= 0x46 # A-F + or 0x61 <= b <= 0x66 + ) # a-f + + +def escape_decode(data, errors="strict"): + if isinstance(data, str): + data = data.encode("latin-1") l = len(data) i = 0 res = bytearray() while i < l: - - if data[i] == '\\': + if data[i] == 0x5C: # '\\' i += 1 if i >= l: raise ValueError("Trailing \\ in string") - else: - if data[i] == '\\': - res += b'\\' - elif data[i] == 'n': - res += b'\n' - elif data[i] == 't': - res += b'\t' - elif data[i] == 'r': - res += b'\r' - elif data[i] == 'b': - res += b'\b' - elif data[i] == '\'': - res += b'\'' - elif data[i] == '\"': - res += b'\"' - elif data[i] == 'f': - res += b'\f' - elif data[i] == 'a': - res += b'\a' - elif data[i] == 'v': - res += b'\v' - elif '0' <= data[i] <= '9': - # emulate a strange wrap-around behavior of CPython: - # \400 is the same as \000 because 0400 == 256 - octal = data[i:i+3] - res.append(int(octal, 8) & 0xFF) - i += 2 - elif data[i] == 'x': - hexa = data[i+1:i+3] - res.append(int(hexa, 16)) + ch = data[i] + if ch == 0x5C: + res.append(0x5C) # \\ + elif ch == 0x27: + res.append(0x27) # \' + elif ch == 0x22: + res.append(0x22) # \" + elif ch == 0x61: + res.append(0x07) # \a + elif ch == 0x62: + res.append(0x08) # \b + elif ch == 0x66: + res.append(0x0C) # \f + elif ch == 0x6E: + res.append(0x0A) # \n + elif ch == 0x72: + res.append(0x0D) # \r + elif ch == 0x74: + res.append(0x09) # \t + elif ch == 0x76: + res.append(0x0B) # \v + elif ch == 0x0A: + pass # \ continuation + elif 0x30 <= ch <= 0x37: # \0-\7 octal + val = ch - 0x30 + if i + 1 < l and 0x30 <= data[i + 1] <= 0x37: + i += 1 + val = (val << 3) | (data[i] - 0x30) + if i + 1 < l and 0x30 <= data[i + 1] <= 0x37: + i += 1 + val = (val << 3) | (data[i] - 0x30) + res.append(val & 0xFF) + elif ch == 0x78: # \x hex + hex_count = 0 + for j in range(1, 3): + if i + j < l and _is_hex_digit(data[i + j]): + hex_count += 1 + else: + break + if hex_count < 2: + if errors == "strict": + raise ValueError("invalid \\x escape at position %d" % (i - 1)) + elif errors == "replace": + res.append(0x3F) # '?' + i += hex_count + else: + res.append(int(bytes(data[i + 1 : i + 3]), 16)) i += 2 + else: + import warnings + + warnings.warn( + '"\\%c" is an invalid escape sequence' % ch + if 0x20 <= ch < 0x7F + else '"\\x%02x" is an invalid escape sequence' % ch, + DeprecationWarning, + stacklevel=2, + ) + res.append(0x5C) + res.append(ch) else: res.append(data[i]) i += 1 - res = bytes(res) - return res, len(res) + return bytes(res), l + -def charmap_decode( data, errors='strict', mapping=None): - """None - """ +def charmap_decode(data, errors="strict", mapping=None): + """None""" res = PyUnicode_DecodeCharmap(data, len(data), mapping, errors) - res = ''.join(res) + res = "".join(res) return res, len(data) -def utf_7_encode( obj, errors='strict'): - """None - """ +def utf_7_encode(obj, errors="strict"): + """None""" res = PyUnicode_EncodeUTF7(obj, len(obj), 0, 0, errors) - res = b''.join(res) + res = b"".join(res) return res, len(obj) -def mbcs_encode( obj, errors='strict'): - """None - """ + +def mbcs_encode(obj, errors="strict"): + """None""" pass + + ## return (PyUnicode_EncodeMBCS( -## (obj), +## (obj), ## len(obj), ## errors), ## len(obj)) - -def ascii_encode( obj, errors='strict'): - """None - """ + +def ascii_encode(obj, errors="strict"): + """None""" res = PyUnicode_EncodeASCII(obj, len(obj), errors) res = bytes(res) return res, len(obj) -def utf_16_encode( obj, errors='strict'): - """None - """ - res = PyUnicode_EncodeUTF16(obj, len(obj), errors, 'native') + +def utf_16_encode(obj, errors="strict"): + """None""" + res = PyUnicode_EncodeUTF16(obj, len(obj), errors, "native") res = bytes(res) return res, len(obj) -def raw_unicode_escape_encode( obj, errors='strict'): - """None - """ + +def raw_unicode_escape_encode(obj, errors="strict"): + """None""" res = PyUnicode_EncodeRawUnicodeEscape(obj, len(obj)) res = bytes(res) return res, len(obj) -def utf_16_le_encode( obj, errors='strict'): - """None - """ - res = PyUnicode_EncodeUTF16(obj, len(obj), errors, 'little') + +def utf_16_le_encode(obj, errors="strict"): + """None""" + res = PyUnicode_EncodeUTF16(obj, len(obj), errors, "little") res = bytes(res) return res, len(obj) -def utf_16_be_encode( obj, errors='strict'): - """None - """ - res = PyUnicode_EncodeUTF16(obj, len(obj), errors, 'big') + +def utf_16_be_encode(obj, errors="strict"): + """None""" + res = PyUnicode_EncodeUTF16(obj, len(obj), errors, "big") res = bytes(res) return res, len(obj) -def utf_16_le_decode( data, errors='strict', byteorder=0, final = 0): - """None - """ - consumed = len(data) - if final: - consumed = 0 - res, consumed, byteorder = PyUnicode_DecodeUTF16Stateful(data, len(data), errors, 'little', final) - res = ''.join(res) + +def utf_16_le_decode(data, errors="strict", final=0): + res, consumed, byteorder = PyUnicode_DecodeUTF16Stateful( + data, len(data), errors, "little", final + ) + res = "".join(res) return res, consumed -def utf_16_be_decode( data, errors='strict', byteorder=0, final = 0): - """None - """ - consumed = len(data) - if final: - consumed = 0 - res, consumed, byteorder = PyUnicode_DecodeUTF16Stateful(data, len(data), errors, 'big', final) - res = ''.join(res) + +def utf_16_be_decode(data, errors="strict", final=0): + res, consumed, byteorder = PyUnicode_DecodeUTF16Stateful( + data, len(data), errors, "big", final + ) + res = "".join(res) return res, consumed +def STORECHAR32(ch, byteorder): + """Store a 32-bit character as 4 bytes in the specified byte order.""" + b0 = ch & 0xFF + b1 = (ch >> 8) & 0xFF + b2 = (ch >> 16) & 0xFF + b3 = (ch >> 24) & 0xFF + if byteorder == "little": + return [b0, b1, b2, b3] + else: # big-endian + return [b3, b2, b1, b0] + + +def PyUnicode_EncodeUTF32(s, size, errors, byteorder="little"): + """Encode a Unicode string to UTF-32.""" + p = [] + bom = sys.byteorder + + if byteorder == "native": + bom = sys.byteorder + # Add BOM for native encoding + p += STORECHAR32(0xFEFF, bom) + + if byteorder == "little": + bom = "little" + elif byteorder == "big": + bom = "big" + + pos = 0 + while pos < len(s): + ch = ord(s[pos]) + if 0xD800 <= ch <= 0xDFFF: + if errors == "surrogatepass": + p += STORECHAR32(ch, bom) + pos += 1 + else: + res, pos = unicode_call_errorhandler( + errors, "utf-32", "surrogates not allowed", s, pos, pos + 1, False + ) + for c in res: + p += STORECHAR32(ord(c), bom) + else: + p += STORECHAR32(ch, bom) + pos += 1 + + return p + + +def utf_32_encode(obj, errors="strict"): + """UTF-32 encoding with BOM.""" + encoded = PyUnicode_EncodeUTF32(obj, len(obj), errors, "native") + return bytes(encoded), len(obj) + + +def utf_32_le_encode(obj, errors="strict"): + """UTF-32 little-endian encoding without BOM.""" + encoded = PyUnicode_EncodeUTF32(obj, len(obj), errors, "little") + return bytes(encoded), len(obj) + + +def utf_32_be_encode(obj, errors="strict"): + """UTF-32 big-endian encoding without BOM.""" + res = PyUnicode_EncodeUTF32(obj, len(obj), errors, "big") + res = bytes(res) + return res, len(obj) + + +def PyUnicode_DecodeUTF32Stateful(data, size, errors, byteorder="little", final=0): + """Decode UTF-32 encoded bytes to Unicode string.""" + if size == 0: + return [], 0, 0 + + result = [] + pos = 0 + aligned_size = (size // 4) * 4 + + while pos + 3 < aligned_size: + if byteorder == "little": + ch = ( + data[pos] + | (data[pos + 1] << 8) + | (data[pos + 2] << 16) + | (data[pos + 3] << 24) + ) + else: # big-endian + ch = ( + (data[pos] << 24) + | (data[pos + 1] << 16) + | (data[pos + 2] << 8) + | data[pos + 3] + ) + + # Validate code point + if ch > 0x10FFFF: + if errors == "strict": + raise UnicodeDecodeError( + "utf-32", + bytes(data), + pos, + pos + 4, + "codepoint not in range(0x110000)", + ) + elif errors == "replace": + result.append("\ufffd") + # 'ignore' - skip this character + pos += 4 + elif 0xD800 <= ch <= 0xDFFF: + if errors == "surrogatepass": + result.append(chr(ch)) + pos += 4 + else: + msg = "code point in surrogate code point range(0xd800, 0xe000)" + res, pos = unicode_call_errorhandler( + errors, "utf-32", msg, data, pos, pos + 4, True + ) + result.append(res) + else: + result.append(chr(ch)) + pos += 4 + + # Handle trailing incomplete bytes + if pos < size: + if final: + res, pos = unicode_call_errorhandler( + errors, "utf-32", "truncated data", data, pos, size, True + ) + if res: + result.append(res) + + return result, pos, 0 + + +def utf_32_decode(data, errors="strict", final=0): + """UTF-32 decoding with BOM detection.""" + if len(data) >= 4: + # Check for BOM + if data[0:4] == b"\xff\xfe\x00\x00": + # UTF-32 LE BOM + res, consumed, _ = PyUnicode_DecodeUTF32Stateful( + data[4:], len(data) - 4, errors, "little", final + ) + res = "".join(res) + return res, consumed + 4 + elif data[0:4] == b"\x00\x00\xfe\xff": + # UTF-32 BE BOM + res, consumed, _ = PyUnicode_DecodeUTF32Stateful( + data[4:], len(data) - 4, errors, "big", final + ) + res = "".join(res) + return res, consumed + 4 + + # Default to little-endian if no BOM + byteorder = "little" if sys.byteorder == "little" else "big" + res, consumed, _ = PyUnicode_DecodeUTF32Stateful( + data, len(data), errors, byteorder, final + ) + res = "".join(res) + return res, consumed + + +def utf_32_le_decode(data, errors="strict", final=0): + """UTF-32 little-endian decoding without BOM.""" + res, consumed, _ = PyUnicode_DecodeUTF32Stateful( + data, len(data), errors, "little", final + ) + res = "".join(res) + return res, consumed + + +def utf_32_be_decode(data, errors="strict", final=0): + """UTF-32 big-endian decoding without BOM.""" + res, consumed, _ = PyUnicode_DecodeUTF32Stateful( + data, len(data), errors, "big", final + ) + res = "".join(res) + return res, consumed # ---------------------------------------------------------------------- @@ -364,9 +649,9 @@ def utf_16_be_decode( data, errors='strict', byteorder=0, final = 0): ##import sys ##""" Python implementation of CPythons builtin unicode codecs. ## -## Generally the functions in this module take a list of characters an returns +## Generally the functions in this module take a list of characters an returns ## a list of characters. -## +## ## For use in the PyPy project""" @@ -376,283 +661,496 @@ def utf_16_be_decode( data, errors='strict', byteorder=0, final = 0): ## 1 - special ## 2 - whitespace (optional) ## 3 - RFC2152 Set O (optional) - + utf7_special = [ - 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 1, 1, 2, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 2, 3, 3, 3, 3, 3, 3, 0, 0, 0, 3, 1, 0, 0, 0, 1, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 3, 0, - 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 3, 3, 3, - 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 1, 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 3, + 3, + 3, + 3, + 3, + 3, + 0, + 0, + 0, + 3, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 3, + 3, + 3, + 3, + 0, + 3, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 3, + 1, + 3, + 3, + 3, + 3, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 3, + 3, + 3, + 1, + 1, ] -unicode_latin1 = [None]*256 +unicode_latin1 = [None] * 256 def SPECIAL(c, encodeO, encodeWS): c = ord(c) - return (c>127 or utf7_special[c] == 1) or \ - (encodeWS and (utf7_special[(c)] == 2)) or \ - (encodeO and (utf7_special[(c)] == 3)) + return ( + (c > 127 or utf7_special[c] == 1) + or (encodeWS and (utf7_special[(c)] == 2)) + or (encodeO and (utf7_special[(c)] == 3)) + ) + + def B64(n): - return bytes([b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"[(n) & 0x3f]]) + return bytes( + [ + b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"[ + (n) & 0x3F + ] + ] + ) + + def B64CHAR(c): - return (c.isalnum() or (c) == b'+' or (c) == b'/') + return c.isalnum() or (c) == b"+" or (c) == b"/" + + def UB64(c): - if (c) == b'+' : - return 62 - elif (c) == b'/': - return 63 - elif (c) >= b'a': - return ord(c) - 71 - elif (c) >= b'A': - return ord(c) - 65 - else: + if (c) == b"+": + return 62 + elif (c) == b"/": + return 63 + elif (c) >= b"a": + return ord(c) - 71 + elif (c) >= b"A": + return ord(c) - 65 + else: return ord(c) + 4 -def ENCODE( ch, bits) : + +def ENCODE(ch, bits): out = [] - while (bits >= 6): - out += B64(ch >> (bits-6)) - bits -= 6 + while bits >= 6: + out += B64(ch >> (bits - 6)) + bits -= 6 return out, bits -def PyUnicode_DecodeUTF7(s, size, errors): - starts = s - errmsg = "" - inShift = 0 - bitsleft = 0 - charsleft = 0 - surrogate = 0 - p = [] - errorHandler = None - exc = None +def _IS_BASE64(ch): + return ( + (ord("A") <= ch <= ord("Z")) + or (ord("a") <= ch <= ord("z")) + or (ord("0") <= ch <= ord("9")) + or ch == ord("+") + or ch == ord("/") + ) - if (size == 0): - return '' + +def _FROM_BASE64(ch): + if ch == ord("+"): + return 62 + if ch == ord("/"): + return 63 + if ch >= ord("a"): + return ch - 71 + if ch >= ord("A"): + return ch - 65 + if ch >= ord("0"): + return ch - ord("0") + 52 + return -1 + + +def _DECODE_DIRECT(ch): + return ch <= 127 and ch != ord("+") + + +def PyUnicode_DecodeUTF7(s, size, errors, final=False): + if size == 0: + return [], 0 + + p = [] + inShift = False + base64bits = 0 + base64buffer = 0 + surrogate = 0 + startinpos = 0 + shiftOutStart = 0 i = 0 + while i < size: - - ch = bytes([s[i]]) - if (inShift): - if ((ch == b'-') or not B64CHAR(ch)): - inShift = 0 + ch = s[i] + if inShift: + if _IS_BASE64(ch): + base64buffer = (base64buffer << 6) | _FROM_BASE64(ch) + base64bits += 6 i += 1 - - while (bitsleft >= 16): - outCh = ((charsleft) >> (bitsleft-16)) & 0xffff - bitsleft -= 16 - - if (surrogate): - ## We have already generated an error for the high surrogate - ## so let's not bother seeing if the low surrogate is correct or not - surrogate = 0 - elif (0xDC00 <= (outCh) and (outCh) <= 0xDFFF): - ## This is a surrogate pair. Unfortunately we can't represent - ## it in a 16-bit character - surrogate = 1 - msg = "code pairs are not supported" - out, x = unicode_call_errorhandler(errors, 'utf-7', msg, s, i-1, i) - p.append(out) - bitsleft = 0 - break + if base64bits >= 16: + outCh = (base64buffer >> (base64bits - 16)) & 0xFFFF + base64bits -= 16 + base64buffer &= (1 << base64bits) - 1 + if surrogate: + if 0xDC00 <= outCh <= 0xDFFF: + ch2 = ( + 0x10000 + + ((surrogate - 0xD800) << 10) + + (outCh - 0xDC00) + ) + p.append(chr(ch2)) + surrogate = 0 + continue + else: + p.append(chr(surrogate)) + surrogate = 0 + if 0xD800 <= outCh <= 0xDBFF: + surrogate = outCh else: - p.append(chr(outCh )) - #p += out - if (bitsleft >= 6): -## /* The shift sequence has a partial character in it. If -## bitsleft < 6 then we could just classify it as padding -## but that is not the case here */ - msg = "partial character in shift sequence" - out, x = unicode_call_errorhandler(errors, 'utf-7', msg, s, i-1, i) - -## /* According to RFC2152 the remaining bits should be zero. We -## choose to signal an error/insert a replacement character -## here so indicate the potential of a misencoded character. */ - -## /* On x86, a << b == a << (b%32) so make sure that bitsleft != 0 */ -## if (bitsleft and (charsleft << (sizeof(charsleft) * 8 - bitsleft))): -## raise UnicodeDecodeError, "non-zero padding bits in shift sequence" - if (ch == b'-') : - if ((i < size) and (s[i] == '-')) : - p += '-' - inShift = 1 - - elif SPECIAL(ch, 0, 0) : - raise UnicodeDecodeError("unexpected special character") - - else: - p.append(chr(ord(ch))) + p.append(chr(outCh)) else: - charsleft = (charsleft << 6) | UB64(ch) - bitsleft += 6 - i += 1 -## /* p, charsleft, bitsleft, surrogate = */ DECODE(p, charsleft, bitsleft, surrogate); - elif ( ch == b'+' ): + inShift = False + if base64bits > 0: + if base64bits >= 6: + i += 1 + errmsg = "partial character in shift sequence" + out, i = unicode_call_errorhandler( + errors, "utf-7", errmsg, s, startinpos, i + ) + p.append(out) + continue + else: + if base64buffer != 0: + i += 1 + errmsg = "non-zero padding bits in shift sequence" + out, i = unicode_call_errorhandler( + errors, "utf-7", errmsg, s, startinpos, i + ) + p.append(out) + continue + if surrogate and _DECODE_DIRECT(ch): + p.append(chr(surrogate)) + surrogate = 0 + if ch == ord("-"): + i += 1 + elif ch == ord("+"): startinpos = i i += 1 - if (i= 6 or (base64bits > 0 and base64buffer != 0): + errmsg = "unterminated shift sequence" + out, i = unicode_call_errorhandler( + errors, "utf-7", errmsg, s, startinpos, size + ) + p.append(out) + + return p, size + + +def _ENCODE_DIRECT(ch, encodeSetO, encodeWhiteSpace): + c = ord(ch) if isinstance(ch, str) else ch + if c > 127: + return False + if utf7_special[c] == 0: + return True + if utf7_special[c] == 2: + return not encodeWhiteSpace + if utf7_special[c] == 3: + return not encodeSetO + return False - if (inShift) : - #XXX This aint right - endinpos = size - raise UnicodeDecodeError("unterminated shift sequence") - - return p def PyUnicode_EncodeUTF7(s, size, encodeSetO, encodeWhiteSpace, errors): - -# /* It might be possible to tighten this worst case */ inShift = False - i = 0 - bitsleft = 0 - charsleft = 0 + base64bits = 0 + base64buffer = 0 out = [] - for ch in s: - if (not inShift) : - if (ch == '+'): - out.append(b'+-') - elif (SPECIAL(ch, encodeSetO, encodeWhiteSpace)): - charsleft = ord(ch) - bitsleft = 16 - out.append(b'+') - p, bitsleft = ENCODE( charsleft, bitsleft) - out.append(p) - inShift = bitsleft > 0 + + for i, ch in enumerate(s): + ch_ord = ord(ch) + if inShift: + if _ENCODE_DIRECT(ch, encodeSetO, encodeWhiteSpace): + # shifting out + if base64bits: + out.append(B64(base64buffer << (6 - base64bits))) + base64buffer = 0 + base64bits = 0 + inShift = False + if B64CHAR(ch) or ch == "-": + out.append(b"-") + out.append(bytes([ch_ord])) else: - out.append(bytes([ord(ch)])) + # encode character in base64 + if ch_ord >= 0x10000: + # split into surrogate pair + hi = 0xD800 | ((ch_ord - 0x10000) >> 10) + lo = 0xDC00 | ((ch_ord - 0x10000) & 0x3FF) + base64bits += 16 + base64buffer = (base64buffer << 16) | hi + while base64bits >= 6: + out.append(B64(base64buffer >> (base64bits - 6))) + base64bits -= 6 + base64buffer &= (1 << base64bits) - 1 if base64bits else 0 + ch_ord = lo + + base64bits += 16 + base64buffer = (base64buffer << 16) | ch_ord + while base64bits >= 6: + out.append(B64(base64buffer >> (base64bits - 6))) + base64bits -= 6 + base64buffer &= (1 << base64bits) - 1 if base64bits else 0 else: - if (not SPECIAL(ch, encodeSetO, encodeWhiteSpace)): - out.append(B64((charsleft) << (6-bitsleft))) - charsleft = 0 - bitsleft = 0 -## /* Characters not in the BASE64 set implicitly unshift the sequence -## so no '-' is required, except if the character is itself a '-' */ - if (B64CHAR(ch) or ch == '-'): - out.append(b'-') - inShift = False - out.append(bytes([ord(ch)])) + if ch == "+": + out.append(b"+-") + elif _ENCODE_DIRECT(ch, encodeSetO, encodeWhiteSpace): + out.append(bytes([ch_ord])) else: - bitsleft += 16 - charsleft = (((charsleft) << 16) | ord(ch)) - p, bitsleft = ENCODE(charsleft, bitsleft) - out.append(p) -## /* If the next character is special then we dont' need to terminate -## the shift sequence. If the next character is not a BASE64 character -## or '-' then the shift sequence will be terminated implicitly and we -## don't have to insert a '-'. */ - - if (bitsleft == 0): - if (i + 1 < size): - ch2 = s[i+1] - - if (SPECIAL(ch2, encodeSetO, encodeWhiteSpace)): - pass - elif (B64CHAR(ch2) or ch2 == '-'): - out.append(b'-') - inShift = False - else: + out.append(b"+") + inShift = True + # encode character in base64 + if ch_ord >= 0x10000: + hi = 0xD800 | ((ch_ord - 0x10000) >> 10) + lo = 0xDC00 | ((ch_ord - 0x10000) & 0x3FF) + base64bits += 16 + base64buffer = (base64buffer << 16) | hi + while base64bits >= 6: + out.append(B64(base64buffer >> (base64bits - 6))) + base64bits -= 6 + base64buffer &= (1 << base64bits) - 1 if base64bits else 0 + ch_ord = lo + + base64bits += 16 + base64buffer = (base64buffer << 16) | ch_ord + while base64bits >= 6: + out.append(B64(base64buffer >> (base64bits - 6))) + base64bits -= 6 + base64buffer &= (1 << base64bits) - 1 if base64bits else 0 + + if base64bits == 0: + if i + 1 < size: + ch2 = s[i + 1] + if _ENCODE_DIRECT(ch2, encodeSetO, encodeWhiteSpace): + if B64CHAR(ch2) or ch2 == "-": + out.append(b"-") inShift = False else: - out.append(b'-') + out.append(b"-") inShift = False - i += 1 - - if (bitsleft): - out.append(B64(charsleft << (6-bitsleft) ) ) - out.append(b'-') + + if base64bits: + out.append(B64(base64buffer << (6 - base64bits))) + if inShift: + out.append(b"-") return out -unicode_empty = '' -def unicodeescape_string(s, size, quotes): +unicode_empty = "" + +def unicodeescape_string(s, size, quotes): p = [] - if (quotes) : - if (s.find('\'') != -1 and s.find('"') == -1): + if quotes: + if s.find("'") != -1 and s.find('"') == -1: p.append(b'"') else: - p.append(b'\'') + p.append(b"'") pos = 0 - while (pos < size): + while pos < size: ch = s[pos] - #/* Escape quotes */ - if (quotes and (ch == p[1] or ch == '\\')): - p.append(b'\\%c' % ord(ch)) + # /* Escape quotes */ + if quotes and (ch == p[1] or ch == "\\"): + p.append(b"\\%c" % ord(ch)) pos += 1 continue -#ifdef Py_UNICODE_WIDE - #/* Map 21-bit characters to '\U00xxxxxx' */ - elif (ord(ch) >= 0x10000): - p.append(b'\\U%08x' % ord(ch)) + # ifdef Py_UNICODE_WIDE + # /* Map 21-bit characters to '\U00xxxxxx' */ + elif ord(ch) >= 0x10000: + p.append(b"\\U%08x" % ord(ch)) pos += 1 - continue -#endif - #/* Map UTF-16 surrogate pairs to Unicode \UXXXXXXXX escapes */ - elif (ord(ch) >= 0xD800 and ord(ch) < 0xDC00): + continue + # endif + # /* Map UTF-16 surrogate pairs to Unicode \UXXXXXXXX escapes */ + elif ord(ch) >= 0xD800 and ord(ch) < 0xDC00: pos += 1 ch2 = s[pos] - - if (ord(ch2) >= 0xDC00 and ord(ch2) <= 0xDFFF): + + if ord(ch2) >= 0xDC00 and ord(ch2) <= 0xDFFF: ucs = (((ord(ch) & 0x03FF) << 10) | (ord(ch2) & 0x03FF)) + 0x00010000 - p.append(b'\\U%08x' % ucs) + p.append(b"\\U%08x" % ucs) pos += 1 continue - - #/* Fall through: isolated surrogates are copied as-is */ + + # /* Fall through: isolated surrogates are copied as-is */ pos -= 1 - - #/* Map 16-bit characters to '\uxxxx' */ - if (ord(ch) >= 256): - p.append(b'\\u%04x' % ord(ch)) - - #/* Map special whitespace to '\t', \n', '\r' */ - elif (ch == '\t'): - p.append(b'\\t') - - elif (ch == '\n'): - p.append(b'\\n') - - elif (ch == '\r'): - p.append(b'\\r') - - elif (ch == '\\'): - p.append(b'\\\\') - - #/* Map non-printable US ASCII to '\xhh' */ - elif (ch < ' ' or ch >= chr(0x7F)) : - p.append(b'\\x%02x' % ord(ch)) - #/* Copy everything else as-is */ + + # /* Map 16-bit characters to '\uxxxx' */ + if ord(ch) >= 256: + p.append(b"\\u%04x" % ord(ch)) + + # /* Map special whitespace to '\t', \n', '\r' */ + elif ch == "\t": + p.append(b"\\t") + + elif ch == "\n": + p.append(b"\\n") + + elif ch == "\r": + p.append(b"\\r") + + elif ch == "\\": + p.append(b"\\\\") + + # /* Map non-printable US ASCII to '\xhh' */ + elif ch < " " or ch >= chr(0x7F): + p.append(b"\\x%02x" % ord(ch)) + # /* Copy everything else as-is */ else: p.append(bytes([ord(ch)])) pos += 1 - if (quotes): + if quotes: p.append(p[0]) return p -def PyUnicode_DecodeASCII(s, size, errors): -# /* ASCII is equivalent to the first 128 ordinals in Unicode. */ - if (size == 1 and ord(s) < 128) : +def PyUnicode_DecodeASCII(s, size, errors): + # /* ASCII is equivalent to the first 128 ordinals in Unicode. */ + if size == 1 and ord(s) < 128: return [chr(ord(s))] - if (size == 0): - return [''] #unicode('') + if size == 0: + return [""] # unicode('') p = [] pos = 0 while pos < len(s): @@ -661,54 +1159,50 @@ def PyUnicode_DecodeASCII(s, size, errors): p += chr(c) pos += 1 else: - res = unicode_call_errorhandler( - errors, "ascii", "ordinal not in range(128)", - s, pos, pos+1) + errors, "ascii", "ordinal not in range(128)", s, pos, pos + 1 + ) p += res[0] pos = res[1] return p -def PyUnicode_EncodeASCII(p, size, errors): +def PyUnicode_EncodeASCII(p, size, errors): return unicode_encode_ucs1(p, size, errors, 128) -def PyUnicode_AsASCIIString(unistr): +def PyUnicode_AsASCIIString(unistr): if not type(unistr) == str: raise TypeError - return PyUnicode_EncodeASCII(str(unistr), - len(str), - None) + return PyUnicode_EncodeASCII(unistr, len(unistr), None) -def PyUnicode_DecodeUTF16Stateful(s, size, errors, byteorder='native', final=True): - bo = 0 #/* assume native ordering by default */ +def PyUnicode_DecodeUTF16Stateful(s, size, errors, byteorder="native", final=True): + bo = 0 # /* assume native ordering by default */ consumed = 0 errmsg = "" - if sys.byteorder == 'little': + if sys.byteorder == "little": ihi = 1 ilo = 0 else: ihi = 0 ilo = 1 - - #/* Unpack UTF-16 encoded data */ + # /* Unpack UTF-16 encoded data */ -## /* Check for BOM marks (U+FEFF) in the input and adjust current -## byte order setting accordingly. In native mode, the leading BOM -## mark is skipped, in all other modes, it is copied to the output -## stream as-is (giving a ZWNBSP character). */ + ## /* Check for BOM marks (U+FEFF) in the input and adjust current + ## byte order setting accordingly. In native mode, the leading BOM + ## mark is skipped, in all other modes, it is copied to the output + ## stream as-is (giving a ZWNBSP character). */ q = 0 p = [] - if byteorder == 'native': - if (size >= 2): + if byteorder == "native": + if size >= 2: bom = (s[ihi] << 8) | s[ilo] -#ifdef BYTEORDER_IS_LITTLE_ENDIAN - if sys.byteorder == 'little': - if (bom == 0xFEFF): + # ifdef BYTEORDER_IS_LITTLE_ENDIAN + if sys.byteorder == "little": + if bom == 0xFEFF: q += 2 bo = -1 elif bom == 0xFFFE: @@ -721,118 +1215,143 @@ def PyUnicode_DecodeUTF16Stateful(s, size, errors, byteorder='native', final=Tru elif bom == 0xFFFE: q += 2 bo = -1 - elif byteorder == 'little': + elif byteorder == "little": bo = -1 else: bo = 1 - - if (size == 0): - return [''], 0, bo - - if (bo == -1): - #/* force LE */ + + if size == 0: + return [""], 0, bo + + if bo == -1: + # /* force LE */ ihi = 1 ilo = 0 - elif (bo == 1): - #/* force BE */ + elif bo == 1: + # /* force BE */ ihi = 0 ilo = 1 - while (q < len(s)): - - #/* remaining bytes at the end? (size should be even) */ - if (len(s)-q<2): + while q < len(s): + # /* remaining bytes at the end? (size should be even) */ + if len(s) - q < 2: if not final: break - errmsg = "truncated data" - startinpos = q - endinpos = len(s) - unicode_call_errorhandler(errors, 'utf-16', errmsg, s, startinpos, endinpos, True) -# /* The remaining input chars are ignored if the callback -## chooses to skip the input */ - - ch = (s[q+ihi] << 8) | s[q+ilo] - q += 2 - - if (ch < 0xD800 or ch > 0xDFFF): + res, q = unicode_call_errorhandler( + errors, "utf-16", "truncated data", s, q, len(s), True + ) + p.append(res) + break + + ch = (s[q + ihi] << 8) | s[q + ilo] + + if ch < 0xD800 or ch > 0xDFFF: p.append(chr(ch)) - continue - - #/* UTF-16 code pair: */ - if (q >= len(s)): - errmsg = "unexpected end of data" - startinpos = q-2 - endinpos = len(s) - unicode_call_errorhandler(errors, 'utf-16', errmsg, s, startinpos, endinpos, True) - - if (0xD800 <= ch and ch <= 0xDBFF): - ch2 = (s[q+ihi] << 8) | s[q+ilo] q += 2 - if (0xDC00 <= ch2 and ch2 <= 0xDFFF): - #ifndef Py_UNICODE_WIDE - if sys.maxunicode < 65536: - p += [chr(ch), chr(ch2)] + continue + + # /* UTF-16 code pair: high surrogate */ + if 0xD800 <= ch <= 0xDBFF: + if q + 4 <= len(s): + ch2 = (s[q + 2 + ihi] << 8) | s[q + 2 + ilo] + if 0xDC00 <= ch2 <= 0xDFFF: + # Valid surrogate pair - always assemble + p.append(chr((((ch & 0x3FF) << 10) | (ch2 & 0x3FF)) + 0x10000)) + q += 4 + continue else: - p.append(chr((((ch & 0x3FF)<<10) | (ch2 & 0x3FF)) + 0x10000)) - #endif + # High surrogate followed by non-low-surrogate + if errors == "surrogatepass": + p.append(chr(ch)) + q += 2 + continue + res, q = unicode_call_errorhandler( + errors, "utf-16", "illegal UTF-16 surrogate", s, q, q + 2, True + ) + p.append(res) + else: + # High surrogate at end of data + if not final: + break + if errors == "surrogatepass": + p.append(chr(ch)) + q += 2 + continue + res, q = unicode_call_errorhandler( + errors, "utf-16", "unexpected end of data", s, q, len(s), True + ) + p.append(res) + else: + # Low surrogate without preceding high surrogate + if errors == "surrogatepass": + p.append(chr(ch)) + q += 2 continue + res, q = unicode_call_errorhandler( + errors, "utf-16", "illegal encoding", s, q, q + 2, True + ) + p.append(res) - else: - errmsg = "illegal UTF-16 surrogate" - startinpos = q-4 - endinpos = startinpos+2 - unicode_call_errorhandler(errors, 'utf-16', errmsg, s, startinpos, endinpos, True) - - errmsg = "illegal encoding" - startinpos = q-2 - endinpos = startinpos+2 - unicode_call_errorhandler(errors, 'utf-16', errmsg, s, startinpos, endinpos, True) - return p, q, bo + # moved out of local scope, especially because it didn't # have any nested variables. + def STORECHAR(CH, byteorder): - hi = (CH >> 8) & 0xff - lo = CH & 0xff - if byteorder == 'little': + hi = (CH >> 8) & 0xFF + lo = CH & 0xFF + if byteorder == "little": return [lo, hi] else: return [hi, lo] -def PyUnicode_EncodeUTF16(s, size, errors, byteorder='little'): -# /* Offsets from p for storing byte pairs in the right order. */ +def PyUnicode_EncodeUTF16(s, size, errors, byteorder="little"): + # /* Offsets from p for storing byte pairs in the right order. */ - p = [] bom = sys.byteorder - if (byteorder == 'native'): - + if byteorder == "native": bom = sys.byteorder p += STORECHAR(0xFEFF, bom) - - if (size == 0): - return "" - if (byteorder == 'little' ): - bom = 'little' - elif (byteorder == 'big'): - bom = 'big' + if byteorder == "little": + bom = "little" + elif byteorder == "big": + bom = "big" - - for c in s: - ch = ord(c) - ch2 = 0 - if (ch >= 0x10000) : - ch2 = 0xDC00 | ((ch-0x10000) & 0x3FF) - ch = 0xD800 | ((ch-0x10000) >> 10) - - p += STORECHAR(ch, bom) - if (ch2): - p += STORECHAR(ch2, bom) + pos = 0 + while pos < len(s): + ch = ord(s[pos]) + if 0xD800 <= ch <= 0xDFFF: + if errors == "surrogatepass": + p += STORECHAR(ch, bom) + pos += 1 + else: + res, pos = unicode_call_errorhandler( + errors, "utf-16", "surrogates not allowed", s, pos, pos + 1, False + ) + for c in res: + cp = ord(c) + cp2 = 0 + if cp >= 0x10000: + cp2 = 0xDC00 | ((cp - 0x10000) & 0x3FF) + cp = 0xD800 | ((cp - 0x10000) >> 10) + p += STORECHAR(cp, bom) + if cp2: + p += STORECHAR(cp2, bom) + else: + ch2 = 0 + if ch >= 0x10000: + ch2 = 0xDC00 | ((ch - 0x10000) & 0x3FF) + ch = 0xD800 | ((ch - 0x10000) >> 10) + p += STORECHAR(ch, bom) + if ch2: + p += STORECHAR(ch2, bom) + pos += 1 return p @@ -840,119 +1359,149 @@ def PyUnicode_EncodeUTF16(s, size, errors, byteorder='little'): def PyUnicode_DecodeMBCS(s, size, errors): pass + def PyUnicode_EncodeMBCS(p, size, errors): pass -def unicode_call_errorhandler(errors, encoding, - reason, input, startinpos, endinpos, decode=True): - + +def unicode_call_errorhandler( + errors, encoding, reason, input, startinpos, endinpos, decode=True +): errorHandler = lookup_error(errors) if decode: - exceptionObject = UnicodeDecodeError(encoding, input, startinpos, endinpos, reason) + exceptionObject = UnicodeDecodeError( + encoding, input, startinpos, endinpos, reason + ) else: - exceptionObject = UnicodeEncodeError(encoding, input, startinpos, endinpos, reason) + exceptionObject = UnicodeEncodeError( + encoding, input, startinpos, endinpos, reason + ) res = errorHandler(exceptionObject) - if isinstance(res, tuple) and isinstance(res[0], str) and isinstance(res[1], int): + if ( + isinstance(res, tuple) + and isinstance(res[0], (str, bytes)) + and isinstance(res[1], int) + ): newpos = res[1] - if (newpos < 0): + if newpos < 0: newpos = len(input) + newpos if newpos < 0 or newpos > len(input): - raise IndexError( "position %d from error handler out of bounds" % newpos) + raise IndexError("position %d from error handler out of bounds" % newpos) return res[0], newpos else: - raise TypeError("encoding error handler must return (unicode, int) tuple, not %s" % repr(res)) + raise TypeError( + "encoding error handler must return (unicode, int) tuple, not %s" + % repr(res) + ) + + +# /* --- Latin-1 Codec ------------------------------------------------------ */ -#/* --- Latin-1 Codec ------------------------------------------------------ */ def PyUnicode_DecodeLatin1(s, size, errors): - #/* Latin-1 is equivalent to the first 256 ordinals in Unicode. */ -## if (size == 1): -## return [PyUnicode_FromUnicode(s, 1)] + # /* Latin-1 is equivalent to the first 256 ordinals in Unicode. */ + ## if (size == 1): + ## return [PyUnicode_FromUnicode(s, 1)] pos = 0 p = [] - while (pos < size): + while pos < size: p += chr(s[pos]) pos += 1 return p + def unicode_encode_ucs1(p, size, errors, limit): - if limit == 256: reason = "ordinal not in range(256)" encoding = "latin-1" else: reason = "ordinal not in range(128)" encoding = "ascii" - - if (size == 0): + + if size == 0: return [] res = bytearray() pos = 0 while pos < len(p): - #for ch in p: + # for ch in p: ch = p[pos] - + if ord(ch) < limit: res.append(ord(ch)) pos += 1 else: - #/* startpos for collecting unencodable chars */ - collstart = pos - collend = pos+1 + # /* startpos for collecting unencodable chars */ + collstart = pos + collend = pos + 1 while collend < len(p) and ord(p[collend]) >= limit: collend += 1 - x = unicode_call_errorhandler(errors, encoding, reason, p, collstart, collend, False) - res += x[0].encode() + x = unicode_call_errorhandler( + errors, encoding, reason, p, collstart, collend, False + ) + replacement = x[0] + if isinstance(replacement, bytes): + res += replacement + else: + res += replacement.encode() pos = x[1] - + return res + def PyUnicode_EncodeLatin1(p, size, errors): res = unicode_encode_ucs1(p, size, errors, 256) return res -hexdigits = [ord(hex(i)[-1]) for i in range(16)]+[ord(hex(i)[-1].upper()) for i in range(10, 16)] + +hexdigits = [ord(hex(i)[-1]) for i in range(16)] + [ + ord(hex(i)[-1].upper()) for i in range(10, 16) +] + def hex_number_end(s, pos, digits): target_end = pos + digits - while pos < target_end and pos < len(s) and s[pos] in hexdigits: + while pos < target_end and pos < len(s) and s[pos] in hexdigits: pos += 1 return pos + def hexescape(s, pos, digits, message, errors): ch = 0 p = [] number_end = hex_number_end(s, pos, digits) if number_end - pos != digits: - x = unicode_call_errorhandler(errors, "unicodeescape", message, s, pos-2, number_end) + x = unicode_call_errorhandler( + errors, "unicodeescape", message, s, pos - 2, number_end + ) p.append(x[0]) pos = x[1] else: - ch = int(s[pos:pos+digits], 16) - #/* when we get here, ch is a 32-bit unicode character */ + ch = int(s[pos : pos + digits], 16) + # /* when we get here, ch is a 32-bit unicode character */ if ch <= sys.maxunicode: p.append(chr(ch)) pos += digits - elif (ch <= 0x10ffff): + elif ch <= 0x10FFFF: ch -= 0x10000 p.append(chr(0xD800 + (ch >> 10))) - p.append(chr(0xDC00 + (ch & 0x03FF))) + p.append(chr(0xDC00 + (ch & 0x03FF))) pos += digits else: message = "illegal Unicode character" - x = unicode_call_errorhandler(errors, "unicodeescape", message, s, pos-2, - pos+digits) + x = unicode_call_errorhandler( + errors, "unicodeescape", message, s, pos - 2, pos + digits + ) p.append(x[0]) pos = x[1] res = p return res, pos + def PyUnicode_DecodeUnicodeEscape(s, size, errors, final): + if size == 0: + return "", 0 - if (size == 0): - return '' - if isinstance(s, str): s = s.encode() @@ -960,131 +1509,168 @@ def PyUnicode_DecodeUnicodeEscape(s, size, errors, final): p = [] pos = 0 - while (pos < size): -## /* Non-escape characters are interpreted as Unicode ordinals */ - if (chr(s[pos]) != '\\') : + while pos < size: + ## /* Non-escape characters are interpreted as Unicode ordinals */ + if s[pos] != ord("\\"): p.append(chr(s[pos])) pos += 1 continue -## /* \ - Escapes */ - else: - pos += 1 - if pos >= len(s): - errmessage = "\\ at end of string" - unicode_call_errorhandler(errors, "unicodeescape", errmessage, s, pos-1, size) - ch = chr(s[pos]) - pos += 1 - ## /* \x escapes */ - if ch == '\n': pass - elif ch == '\\': p += '\\' - elif ch == '\'': p += '\'' - elif ch == '\"': p += '\"' - elif ch == 'b' : p += '\b' - elif ch == 'f' : p += '\014' #/* FF */ - elif ch == 't' : p += '\t' - elif ch == 'n' : p += '\n' - elif ch == 'r' : p += '\r' - elif ch == 'v' : p += '\013' #break; /* VT */ - elif ch == 'a' : p += '\007' # break; /* BEL, not classic C */ - elif '0' <= ch <= '7': - x = ord(ch) - ord('0') - if pos < size: - ch = chr(s[pos]) - if '0' <= ch <= '7': - pos += 1 - x = (x<<3) + ord(ch) - ord('0') - if pos < size: - ch = chr(s[pos]) - if '0' <= ch <= '7': - pos += 1 - x = (x<<3) + ord(ch) - ord('0') - p.append(chr(x)) - ## /* hex escapes */ - ## /* \xXX */ - elif ch == 'x': + ## /* \ - Escapes */ + escape_start = pos + pos += 1 + if pos >= size: + if not final: + pos = escape_start + break + errmessage = "\\ at end of string" + unicode_call_errorhandler( + errors, "unicodeescape", errmessage, s, pos - 1, size + ) + break + ch = chr(s[pos]) + pos += 1 + ## /* \x escapes */ + if ch == "\n": + pass + elif ch == "\\": + p += "\\" + elif ch == "'": + p += "'" + elif ch == '"': + p += '"' + elif ch == "b": + p += "\b" + elif ch == "f": + p += "\014" # /* FF */ + elif ch == "t": + p += "\t" + elif ch == "n": + p += "\n" + elif ch == "r": + p += "\r" + elif ch == "v": + p += "\013" # break; /* VT */ + elif ch == "a": + p += "\007" # break; /* BEL, not classic C */ + elif "0" <= ch <= "7": + x = ord(ch) - ord("0") + if pos < size: + ch = chr(s[pos]) + if "0" <= ch <= "7": + pos += 1 + x = (x << 3) + ord(ch) - ord("0") + if pos < size: + ch = chr(s[pos]) + if "0" <= ch <= "7": + pos += 1 + x = (x << 3) + ord(ch) - ord("0") + p.append(chr(x)) + ## /* hex escapes */ + ## /* \xXX */ + elif ch in ("x", "u", "U"): + if ch == "x": digits = 2 message = "truncated \\xXX escape" - x = hexescape(s, pos, digits, message, errors) - p += x[0] - pos = x[1] - - # /* \uXXXX */ - elif ch == 'u': + elif ch == "u": digits = 4 message = "truncated \\uXXXX escape" + else: + digits = 8 + message = "truncated \\UXXXXXXXX escape" + number_end = hex_number_end(s, pos, digits) + if number_end - pos != digits: + if not final: + pos = escape_start + break x = hexescape(s, pos, digits, message, errors) p += x[0] pos = x[1] - - # /* \UXXXXXXXX */ - elif ch == 'U': - digits = 8 - message = "truncated \\UXXXXXXXX escape" + else: x = hexescape(s, pos, digits, message, errors) p += x[0] pos = x[1] -## /* \N{name} */ - elif ch == 'N': - message = "malformed \\N character escape" - # pos += 1 - look = pos - try: - import unicodedata - except ImportError: - message = "\\N escapes not supported (can't load unicodedata module)" - unicode_call_errorhandler(errors, "unicodeescape", message, s, pos-1, size) - if look < size and chr(s[look]) == '{': - #/* look for the closing brace */ - while (look < size and chr(s[look]) != '}'): - look += 1 - if (look > pos+1 and look < size and chr(s[look]) == '}'): - #/* found a name. look it up in the unicode database */ - message = "unknown Unicode character name" - st = s[pos+1:look] - try: - chr_codec = unicodedata.lookup("%s" % st) - except LookupError as e: - x = unicode_call_errorhandler(errors, "unicodeescape", message, s, pos-1, look+1) - else: - x = chr_codec, look + 1 - p.append(x[0]) - pos = x[1] - else: - x = unicode_call_errorhandler(errors, "unicodeescape", message, s, pos-1, look+1) - else: - x = unicode_call_errorhandler(errors, "unicodeescape", message, s, pos-1, look+1) + ## /* \N{name} */ + elif ch == "N": + message = "malformed \\N character escape" + look = pos + try: + import unicodedata + except ImportError: + message = "\\N escapes not supported (can't load unicodedata module)" + unicode_call_errorhandler( + errors, "unicodeescape", message, s, pos - 1, size + ) + continue + if look < size and chr(s[look]) == "{": + # /* look for the closing brace */ + while look < size and chr(s[look]) != "}": + look += 1 + if look > pos + 1 and look < size and chr(s[look]) == "}": + # /* found a name. look it up in the unicode database */ + message = "unknown Unicode character name" + st = s[pos + 1 : look] + try: + chr_codec = unicodedata.lookup("%s" % st) + except LookupError as e: + x = unicode_call_errorhandler( + errors, "unicodeescape", message, s, pos - 1, look + 1 + ) + else: + x = chr_codec, look + 1 + p.append(x[0]) + pos = x[1] + else: + if not final: + pos = escape_start + break + x = unicode_call_errorhandler( + errors, "unicodeescape", message, s, pos - 1, look + 1 + ) + p.append(x[0]) + pos = x[1] else: - if not found_invalid_escape: - found_invalid_escape = True - warnings.warn("invalid escape sequence '\\%c'" % ch, DeprecationWarning, 2) - p.append('\\') - p.append(ch) - return p + if not final: + pos = escape_start + break + x = unicode_call_errorhandler( + errors, "unicodeescape", message, s, pos - 1, look + 1 + ) + p.append(x[0]) + pos = x[1] + else: + if not found_invalid_escape: + found_invalid_escape = True + warnings.warn( + "invalid escape sequence '\\%c'" % ch, DeprecationWarning, 2 + ) + p.append("\\") + p.append(ch) + return p, pos + def PyUnicode_EncodeRawUnicodeEscape(s, size): - - if (size == 0): - return b'' + if size == 0: + return b"" p = bytearray() for ch in s: -# /* Map 32-bit characters to '\Uxxxxxxxx' */ - if (ord(ch) >= 0x10000): - p += b'\\U%08x' % ord(ch) - elif (ord(ch) >= 256) : -# /* Map 16-bit characters to '\uxxxx' */ - p += b'\\u%04x' % (ord(ch)) -# /* Copy everything else as-is */ + # /* Map 32-bit characters to '\Uxxxxxxxx' */ + if ord(ch) >= 0x10000: + p += b"\\U%08x" % ord(ch) + elif ord(ch) >= 256: + # /* Map 16-bit characters to '\uxxxx' */ + p += b"\\u%04x" % (ord(ch)) + # /* Copy everything else as-is */ else: p.append(ord(ch)) - - #p += '\0' + + # p += '\0' return p -def charmapencode_output(c, mapping): +def charmapencode_output(c, mapping): rep = mapping[c] - if isinstance(rep, int) or isinstance(rep, int): + if isinstance(rep, int): if rep < 256: return [rep] else: @@ -1098,144 +1684,156 @@ def charmapencode_output(c, mapping): else: raise TypeError("character mapping must return integer, None or str") -def PyUnicode_EncodeCharmap(p, size, mapping='latin-1', errors='strict'): -## /* the following variable is used for caching string comparisons -## * -1=not initialized, 0=unknown, 1=strict, 2=replace, -## * 3=ignore, 4=xmlcharrefreplace */ +def PyUnicode_EncodeCharmap(p, size, mapping="latin-1", errors="strict"): + ## /* the following variable is used for caching string comparisons + ## * -1=not initialized, 0=unknown, 1=strict, 2=replace, + ## * 3=ignore, 4=xmlcharrefreplace */ -# /* Default to Latin-1 */ - if mapping == 'latin-1': + # /* Default to Latin-1 */ + if mapping == "latin-1": return PyUnicode_EncodeLatin1(p, size, errors) - if (size == 0): - return b'' + if size == 0: + return b"" inpos = 0 res = [] - while (inpos", p, inpos, inpos+1, False) - try: - for y in x[0]: - res += charmapencode_output(ord(y), mapping) - except KeyError: - raise UnicodeEncodeError("charmap", p, inpos, inpos+1, - "character maps to ") + x = unicode_call_errorhandler( + errors, + "charmap", + "character maps to ", + p, + inpos, + inpos + 1, + False, + ) + replacement = x[0] + if isinstance(replacement, bytes): + res += list(replacement) + else: + try: + for y in replacement: + res += charmapencode_output(ord(y), mapping) + except KeyError: + raise UnicodeEncodeError( + "charmap", p, inpos, inpos + 1, "character maps to " + ) inpos += 1 return res -def PyUnicode_DecodeCharmap(s, size, mapping, errors): -## /* Default to Latin-1 */ - if (mapping == None): +def PyUnicode_DecodeCharmap(s, size, mapping, errors): + ## /* Default to Latin-1 */ + if mapping == None: return PyUnicode_DecodeLatin1(s, size, errors) - if (size == 0): - return '' + if size == 0: + return "" p = [] inpos = 0 - while (inpos< len(s)): - - #/* Get mapping (char ordinal -> integer, Unicode char or None) */ + while inpos < len(s): + # /* Get mapping (char ordinal -> integer, Unicode char or None) */ ch = s[inpos] try: x = mapping[ch] if isinstance(x, int): - if x < 65536: + if x == 0xFFFE: + raise KeyError + if 0 <= x <= 0x10FFFF: p += chr(x) else: - raise TypeError("character mapping must be in range(65536)") + raise TypeError( + "character mapping must be in range(0x%x)" % (0x110000,) + ) elif isinstance(x, str): + if len(x) == 1 and x == "\ufffe": + raise KeyError p += x - elif not x: + elif x is None: raise KeyError else: raise TypeError - except KeyError: - x = unicode_call_errorhandler(errors, "charmap", - "character maps to ", s, inpos, inpos+1) + except (KeyError, IndexError): + x = unicode_call_errorhandler( + errors, "charmap", "character maps to ", s, inpos, inpos + 1 + ) p += x[0] inpos += 1 return p -def PyUnicode_DecodeRawUnicodeEscape(s, size, errors, final): - if (size == 0): - return '' +def PyUnicode_DecodeRawUnicodeEscape(s, size, errors, final): + if size == 0: + return "", 0 if isinstance(s, str): s = s.encode() pos = 0 p = [] - while (pos < len(s)): - ch = chr(s[pos]) - #/* Non-escape characters are interpreted as Unicode ordinals */ - if (ch != '\\'): - p.append(ch) + while pos < len(s): + # /* Non-escape characters are interpreted as Unicode ordinals */ + if s[pos] != ord("\\"): + p.append(chr(s[pos])) pos += 1 - continue + continue startinpos = pos -## /* \u-escapes are only interpreted iff the number of leading -## backslashes is odd */ + p_len_before = len(p) + ## /* \u-escapes are only interpreted iff the number of leading + ## backslashes is odd */ bs = pos while pos < size: - if (s[pos] != ord('\\')): + if s[pos] != ord("\\"): break p.append(chr(s[pos])) pos += 1 - - if (pos >= size): + + if pos >= size: + if not final: + del p[p_len_before:] + pos = startinpos break - if (((pos - bs) & 1) == 0 or - (s[pos] != ord('u') and s[pos] != ord('U'))) : + if ((pos - bs) & 1) == 0 or (s[pos] != ord("u") and s[pos] != ord("U")): p.append(chr(s[pos])) pos += 1 continue - + p.pop(-1) - if s[pos] == ord('u'): - count = 4 - else: - count = 8 + count = 4 if s[pos] == ord("u") else 8 pos += 1 - #/* \uXXXX with 4 hex digits, \Uxxxxxxxx with 8 */ + # /* \uXXXX with 4 hex digits, \Uxxxxxxxx with 8 */ number_end = hex_number_end(s, pos, count) if number_end - pos != count: + if not final: + del p[p_len_before:] + pos = startinpos + break res = unicode_call_errorhandler( - errors, "rawunicodeescape", "truncated \\uXXXX", - s, pos-2, number_end) + errors, "rawunicodeescape", "truncated \\uXXXX", s, pos - 2, number_end + ) p.append(res[0]) pos = res[1] else: - x = int(s[pos:pos+count], 16) - #ifndef Py_UNICODE_WIDE - if sys.maxunicode > 0xffff: - if (x > sys.maxunicode): - res = unicode_call_errorhandler( - errors, "rawunicodeescape", "\\Uxxxxxxxx out of range", - s, pos-2, pos+count) - pos = res[1] - p.append(res[0]) - else: - p.append(chr(x)) - pos += count + x = int(s[pos : pos + count], 16) + if x > sys.maxunicode: + res = unicode_call_errorhandler( + errors, + "rawunicodeescape", + "\\Uxxxxxxxx out of range", + s, + pos - 2, + pos + count, + ) + pos = res[1] + p.append(res[0]) else: - if (x > 0x10000): - res = unicode_call_errorhandler( - errors, "rawunicodeescape", "\\Uxxxxxxxx out of range", - s, pos-2, pos+count) - pos = res[1] - p.append(res[0]) - - #endif - else: - p.append(chr(x)) - pos += count + p.append(chr(x)) + pos += count - return p + return p, pos diff --git a/Lib/_pydatetime.py b/Lib/_pydatetime.py index 38e1f764f00..70251dbb653 100644 --- a/Lib/_pydatetime.py +++ b/Lib/_pydatetime.py @@ -1,12 +1,10 @@ -"""Concrete date/time and related types. - -See http://www.iana.org/time-zones/repository/tz-link.html for -time zone and DST data sources. -""" +"""Pure Python implementation of the datetime module.""" __all__ = ("date", "datetime", "time", "timedelta", "timezone", "tzinfo", "MINYEAR", "MAXYEAR", "UTC") +__name__ = "datetime" + import time as _time import math as _math @@ -18,10 +16,10 @@ def _cmp(x, y): def _get_class_module(self): module_name = self.__class__.__module__ - if module_name == '_pydatetime': - return 'datetime' + if module_name == 'datetime': + return 'datetime.' else: - return module_name + return '' MINYEAR = 1 MAXYEAR = 9999 @@ -64,14 +62,14 @@ def _days_in_month(year, month): def _days_before_month(year, month): "year, month -> number of days in year preceding first day of month." - assert 1 <= month <= 12, 'month must be in 1..12' + assert 1 <= month <= 12, f"month must be in 1..12, not {month}" return _DAYS_BEFORE_MONTH[month] + (month > 2 and _is_leap(year)) def _ymd2ord(year, month, day): "year, month, day -> ordinal, considering 01-Jan-0001 as day 1." - assert 1 <= month <= 12, 'month must be in 1..12' + assert 1 <= month <= 12, f"month must be in 1..12, not {month}" dim = _days_in_month(year, month) - assert 1 <= day <= dim, ('day must be in 1..%d' % dim) + assert 1 <= day <= dim, f"day must be in 1..{dim}, not {day}" return (_days_before_year(year) + _days_before_month(year, month) + day) @@ -204,6 +202,17 @@ def _format_offset(off, sep=':'): s += '.%06d' % ss.microseconds return s +_normalize_century = None +def _need_normalize_century(): + global _normalize_century + if _normalize_century is None: + try: + _normalize_century = ( + _time.strftime("%Y", (99, 1, 1, 0, 0, 0, 0, 1, 0)) != "0099") + except ValueError: + _normalize_century = True + return _normalize_century + # Correctly substitute for %z and %Z escapes in strftime formats. def _wrap_strftime(object, format, timetuple): # Don't call utcoffset() or tzname() unless actually needed. @@ -261,6 +270,20 @@ def _wrap_strftime(object, format, timetuple): # strftime is going to have at this: escape % Zreplace = s.replace('%', '%%') newformat.append(Zreplace) + # Note that datetime(1000, 1, 1).strftime('%G') == '1000' so + # year 1000 for %G can go on the fast path. + elif ((ch in 'YG' or ch in 'FC') and + object.year < 1000 and _need_normalize_century()): + if ch == 'G': + year = int(_time.strftime("%G", timetuple)) + else: + year = object.year + if ch == 'C': + push('{:02}'.format(year // 100)) + else: + push('{:04}'.format(year)) + if ch == 'F': + push('-{:02}-{:02}'.format(*timetuple[1:3])) else: push('%') push(ch) @@ -399,7 +422,7 @@ def _parse_hh_mm_ss_ff(tstr): if pos < len_str: if tstr[pos] not in '.,': - raise ValueError("Invalid microsecond component") + raise ValueError("Invalid microsecond separator") else: pos += 1 if not all(map(_is_ascii_digit, tstr[pos:])): @@ -430,6 +453,17 @@ def _parse_isoformat_time(tstr): time_comps = _parse_hh_mm_ss_ff(timestr) + hour, minute, second, microsecond = time_comps + became_next_day = False + error_from_components = False + if (hour == 24): + if all(time_comp == 0 for time_comp in time_comps[1:]): + hour = 0 + time_comps[0] = hour + became_next_day = True + else: + error_from_components = True + tzi = None if tz_pos == len_str and tstr[-1] == 'Z': tzi = timezone.utc @@ -445,7 +479,7 @@ def _parse_isoformat_time(tstr): # HH:MM:SS len: 8 # HH:MM:SS.f+ len: 10+ - if len(tzstr) in (0, 1, 3): + if len(tzstr) in (0, 1, 3) or tstr[tz_pos-1] == 'Z': raise ValueError("Malformed time zone string") tz_comps = _parse_hh_mm_ss_ff(tzstr) @@ -462,13 +496,13 @@ def _parse_isoformat_time(tstr): time_comps.append(tzi) - return time_comps + return time_comps, became_next_day, error_from_components # tuple[int, int, int] -> tuple[int, int, int] version of date.fromisocalendar def _isoweek_to_gregorian(year, week, day): # Year is bounded this way because 9999-12-31 is (9999, 52, 5) if not MINYEAR <= year <= MAXYEAR: - raise ValueError(f"Year is out of range: {year}") + raise ValueError(f"year must be in {MINYEAR}..{MAXYEAR}, not {year}") if not 0 < week < 53: out_of_range = True @@ -501,7 +535,7 @@ def _isoweek_to_gregorian(year, week, day): def _check_tzname(name): if name is not None and not isinstance(name, str): raise TypeError("tzinfo.tzname() must return None or string, " - "not '%s'" % type(name)) + f"not {type(name).__name__!r}") # name is the offset-producing method, "utcoffset" or "dst". # offset is what it returned. @@ -514,24 +548,24 @@ def _check_utc_offset(name, offset): if offset is None: return if not isinstance(offset, timedelta): - raise TypeError("tzinfo.%s() must return None " - "or timedelta, not '%s'" % (name, type(offset))) + raise TypeError(f"tzinfo.{name}() must return None " + f"or timedelta, not {type(offset).__name__!r}") if not -timedelta(1) < offset < timedelta(1): - raise ValueError("%s()=%s, must be strictly between " - "-timedelta(hours=24) and timedelta(hours=24)" % - (name, offset)) + raise ValueError("offset must be a timedelta " + "strictly between -timedelta(hours=24) and " + f"timedelta(hours=24), not {offset!r}") def _check_date_fields(year, month, day): year = _index(year) month = _index(month) day = _index(day) if not MINYEAR <= year <= MAXYEAR: - raise ValueError('year must be in %d..%d' % (MINYEAR, MAXYEAR), year) + raise ValueError(f"year must be in {MINYEAR}..{MAXYEAR}, not {year}") if not 1 <= month <= 12: - raise ValueError('month must be in 1..12', month) + raise ValueError(f"month must be in 1..12, not {month}") dim = _days_in_month(year, month) if not 1 <= day <= dim: - raise ValueError('day must be in 1..%d' % dim, day) + raise ValueError(f"day {day} must be in range 1..{dim} for month {month} in year {year}") return year, month, day def _check_time_fields(hour, minute, second, microsecond, fold): @@ -540,20 +574,23 @@ def _check_time_fields(hour, minute, second, microsecond, fold): second = _index(second) microsecond = _index(microsecond) if not 0 <= hour <= 23: - raise ValueError('hour must be in 0..23', hour) + raise ValueError(f"hour must be in 0..23, not {hour}") if not 0 <= minute <= 59: - raise ValueError('minute must be in 0..59', minute) + raise ValueError(f"minute must be in 0..59, not {minute}") if not 0 <= second <= 59: - raise ValueError('second must be in 0..59', second) + raise ValueError(f"second must be in 0..59, not {second}") if not 0 <= microsecond <= 999999: - raise ValueError('microsecond must be in 0..999999', microsecond) + raise ValueError(f"microsecond must be in 0..999999, not {microsecond}") if fold not in (0, 1): - raise ValueError('fold must be either 0 or 1', fold) + raise ValueError(f"fold must be either 0 or 1, not {fold}") return hour, minute, second, microsecond, fold def _check_tzinfo_arg(tz): if tz is not None and not isinstance(tz, tzinfo): - raise TypeError("tzinfo argument must be None or of a tzinfo subclass") + raise TypeError( + "tzinfo argument must be None or of a tzinfo subclass, " + f"not {type(tz).__name__!r}" + ) def _divide_and_round(a, b): """divide a by b and round result to the nearest integer @@ -607,7 +644,19 @@ def __new__(cls, days=0, seconds=0, microseconds=0, # guide the C implementation; it's way more convoluted than speed- # ignoring auto-overflow-to-long idiomatic Python could be. - # XXX Check that all inputs are ints or floats. + for name, value in ( + ("days", days), + ("seconds", seconds), + ("microseconds", microseconds), + ("milliseconds", milliseconds), + ("minutes", minutes), + ("hours", hours), + ("weeks", weeks) + ): + if not isinstance(value, (int, float)): + raise TypeError( + f"unsupported type for timedelta {name} component: {type(value).__name__}" + ) # Final values, all integer. # s and us fit in 32-bit signed ints; d isn't bounded. @@ -708,9 +757,9 @@ def __repr__(self): args.append("microseconds=%d" % self._microseconds) if not args: args.append('0') - return "%s.%s(%s)" % (_get_class_module(self), - self.__class__.__qualname__, - ', '.join(args)) + return "%s%s(%s)" % (_get_class_module(self), + self.__class__.__qualname__, + ', '.join(args)) def __str__(self): mm, ss = divmod(self._seconds, 60) @@ -907,6 +956,7 @@ class date: fromtimestamp() today() fromordinal() + strptime() Operators: @@ -989,8 +1039,12 @@ def fromordinal(cls, n): @classmethod def fromisoformat(cls, date_string): """Construct a date from a string in ISO 8601 format.""" + if not isinstance(date_string, str): - raise TypeError('fromisoformat: argument must be str') + raise TypeError('Argument must be a str') + + if not date_string.isascii(): + raise ValueError('Argument must be an ASCII str') if len(date_string) not in (7, 8, 10): raise ValueError(f'Invalid isoformat string: {date_string!r}') @@ -1007,6 +1061,12 @@ def fromisocalendar(cls, year, week, day): This is the inverse of the date.isocalendar() function""" return cls(*_isoweek_to_gregorian(year, week, day)) + @classmethod + def strptime(cls, date_string, format): + """Parse a date string according to the given format (like time.strptime()).""" + import _strptime + return _strptime._strptime_datetime_date(cls, date_string, format) + # Conversions to string def __repr__(self): @@ -1016,11 +1076,11 @@ def __repr__(self): >>> repr(d) 'datetime.date(2010, 1, 1)' """ - return "%s.%s(%d, %d, %d)" % (_get_class_module(self), - self.__class__.__qualname__, - self._year, - self._month, - self._day) + return "%s%s(%d, %d, %d)" % (_get_class_module(self), + self.__class__.__qualname__, + self._year, + self._month, + self._day) # XXX These shouldn't depend on time.localtime(), because that # clips the usable dates to [1970 .. 2038). At least ctime() is # easily done without using strftime() -- that's better too because @@ -1327,6 +1387,7 @@ class time: Constructors: __new__() + strptime() Operators: @@ -1385,6 +1446,12 @@ def __new__(cls, hour=0, minute=0, second=0, microsecond=0, tzinfo=None, *, fold self._fold = fold return self + @classmethod + def strptime(cls, date_string, format): + """string, format -> new time parsed from a string (like time.strptime()).""" + import _strptime + return _strptime._strptime_datetime_time(cls, date_string, format) + # Read-only field accessors @property def hour(self): @@ -1513,7 +1580,7 @@ def __repr__(self): s = ", %d" % self._second else: s = "" - s= "%s.%s(%d, %d%s)" % (_get_class_module(self), + s = "%s%s(%d, %d%s)" % (_get_class_module(self), self.__class__.__qualname__, self._hour, self._minute, s) if self._tzinfo is not None: @@ -1555,7 +1622,7 @@ def fromisoformat(cls, time_string): time_string = time_string.removeprefix('T') try: - return cls(*_parse_isoformat_time(time_string)) + return cls(*_parse_isoformat_time(time_string)[0]) except Exception: raise ValueError(f'Invalid isoformat string: {time_string!r}') @@ -1869,10 +1936,27 @@ def fromisoformat(cls, date_string): if tstr: try: - time_components = _parse_isoformat_time(tstr) + time_components, became_next_day, error_from_components = _parse_isoformat_time(tstr) except ValueError: raise ValueError( f'Invalid isoformat string: {date_string!r}') from None + else: + if error_from_components: + raise ValueError("minute, second, and microsecond must be 0 when hour is 24") + + if became_next_day: + year, month, day = date_components + # Only wrap day/month when it was previously valid + if month <= 12 and day <= (days_in_month := _days_in_month(year, month)): + # Calculate midnight of the next day + day += 1 + if day > days_in_month: + day = 1 + month += 1 + if month > 12: + month = 1 + year += 1 + date_components = [year, month, day] else: time_components = [0, 0, 0, 0, None] @@ -2072,9 +2156,9 @@ def __repr__(self): del L[-1] if L[-1] == 0: del L[-1] - s = "%s.%s(%s)" % (_get_class_module(self), - self.__class__.__qualname__, - ", ".join(map(str, L))) + s = "%s%s(%s)" % (_get_class_module(self), + self.__class__.__qualname__, + ", ".join(map(str, L))) if self._tzinfo is not None: assert s[-1:] == ")" s = s[:-1] + ", tzinfo=%r" % self._tzinfo + ")" @@ -2091,7 +2175,7 @@ def __str__(self): def strptime(cls, date_string, format): 'string, format -> new datetime parsed from a string (like time.strptime()).' import _strptime - return _strptime._strptime_datetime(cls, date_string, format) + return _strptime._strptime_datetime_datetime(cls, date_string, format) def utcoffset(self): """Return the timezone offset as timedelta positive east of UTC (negative west of @@ -2331,7 +2415,7 @@ def __new__(cls, offset, name=_Omitted): if not cls._minoffset <= offset <= cls._maxoffset: raise ValueError("offset must be a timedelta " "strictly between -timedelta(hours=24) and " - "timedelta(hours=24).") + f"timedelta(hours=24), not {offset!r}") return cls._create(offset, name) def __init_subclass__(cls): @@ -2371,12 +2455,12 @@ def __repr__(self): if self is self.utc: return 'datetime.timezone.utc' if self._name is None: - return "%s.%s(%r)" % (_get_class_module(self), - self.__class__.__qualname__, - self._offset) - return "%s.%s(%r, %r)" % (_get_class_module(self), - self.__class__.__qualname__, - self._offset, self._name) + return "%s%s(%r)" % (_get_class_module(self), + self.__class__.__qualname__, + self._offset) + return "%s%s(%r, %r)" % (_get_class_module(self), + self.__class__.__qualname__, + self._offset, self._name) def __str__(self): return self.tzname(None) diff --git a/Lib/_pydecimal.py b/Lib/_pydecimal.py index ff80180a79e..97a629fe92c 100644 --- a/Lib/_pydecimal.py +++ b/Lib/_pydecimal.py @@ -38,10 +38,10 @@ 'ROUND_FLOOR', 'ROUND_UP', 'ROUND_HALF_DOWN', 'ROUND_05UP', # Functions for manipulating contexts - 'setcontext', 'getcontext', 'localcontext', + 'setcontext', 'getcontext', 'localcontext', 'IEEEContext', # Limits for the C version for compatibility - 'MAX_PREC', 'MAX_EMAX', 'MIN_EMIN', 'MIN_ETINY', + 'MAX_PREC', 'MAX_EMAX', 'MIN_EMIN', 'MIN_ETINY', 'IEEE_CONTEXT_MAX_BITS', # C version: compile time choice that enables the thread local context (deprecated, now always true) 'HAVE_THREADS', @@ -83,10 +83,12 @@ MAX_PREC = 999999999999999999 MAX_EMAX = 999999999999999999 MIN_EMIN = -999999999999999999 + IEEE_CONTEXT_MAX_BITS = 512 else: MAX_PREC = 425000000 MAX_EMAX = 425000000 MIN_EMIN = -425000000 + IEEE_CONTEXT_MAX_BITS = 256 MIN_ETINY = MIN_EMIN - (MAX_PREC-1) @@ -417,6 +419,27 @@ def sin(x): return ctx_manager +def IEEEContext(bits, /): + """ + Return a context object initialized to the proper values for one of the + IEEE interchange formats. The argument must be a multiple of 32 and less + than IEEE_CONTEXT_MAX_BITS. + """ + if bits <= 0 or bits > IEEE_CONTEXT_MAX_BITS or bits % 32: + raise ValueError("argument must be a multiple of 32, " + f"with a maximum of {IEEE_CONTEXT_MAX_BITS}") + + ctx = Context() + ctx.prec = 9 * (bits//32) - 2 + ctx.Emax = 3 * (1 << (bits//16 + 3)) + ctx.Emin = 1 - ctx.Emax + ctx.rounding = ROUND_HALF_EVEN + ctx.clamp = 1 + ctx.traps = dict.fromkeys(_signals, False) + + return ctx + + ##### Decimal class ####################################################### # Do not subclass Decimal from numbers.Real and do not register it as such @@ -582,6 +605,21 @@ def __new__(cls, value="0", context=None): raise TypeError("Cannot convert %r to Decimal" % value) + @classmethod + def from_number(cls, number): + """Converts a real number to a decimal number, exactly. + + >>> Decimal.from_number(314) # int + Decimal('314') + >>> Decimal.from_number(0.1) # float + Decimal('0.1000000000000000055511151231257827021181583404541015625') + >>> Decimal.from_number(Decimal('3.14')) # another decimal instance + Decimal('3.14') + """ + if isinstance(number, (int, Decimal, float)): + return cls(number) + raise TypeError("Cannot convert %r to Decimal" % number) + @classmethod def from_float(cls, f): """Converts a float to a decimal number, exactly. @@ -2425,12 +2463,12 @@ def __pow__(self, other, modulo=None, context=None): return ans - def __rpow__(self, other, context=None): + def __rpow__(self, other, modulo=None, context=None): """Swaps self/other and returns __pow__.""" other = _convert_other(other) if other is NotImplemented: return other - return other.__pow__(self, context=context) + return other.__pow__(self, modulo, context=context) def normalize(self, context=None): """Normalize- strip trailing 0s, change anything equal to 0 to 0e0""" @@ -3302,7 +3340,10 @@ def _fill_logical(self, context, opa, opb): return opa, opb def logical_and(self, other, context=None): - """Applies an 'and' operation between self and other's digits.""" + """Applies an 'and' operation between self and other's digits. + + Both self and other must be logical numbers. + """ if context is None: context = getcontext() @@ -3319,14 +3360,20 @@ def logical_and(self, other, context=None): return _dec_from_triple(0, result.lstrip('0') or '0', 0) def logical_invert(self, context=None): - """Invert all its digits.""" + """Invert all its digits. + + The self must be logical number. + """ if context is None: context = getcontext() return self.logical_xor(_dec_from_triple(0,'1'*context.prec,0), context) def logical_or(self, other, context=None): - """Applies an 'or' operation between self and other's digits.""" + """Applies an 'or' operation between self and other's digits. + + Both self and other must be logical numbers. + """ if context is None: context = getcontext() @@ -3343,7 +3390,10 @@ def logical_or(self, other, context=None): return _dec_from_triple(0, result.lstrip('0') or '0', 0) def logical_xor(self, other, context=None): - """Applies an 'xor' operation between self and other's digits.""" + """Applies an 'xor' operation between self and other's digits. + + Both self and other must be logical numbers. + """ if context is None: context = getcontext() @@ -6058,7 +6108,7 @@ def _convert_for_comparison(self, other, equality_op=False): (?P\d*) # with (possibly empty) diagnostic info. ) # \s* - \Z + \z """, re.VERBOSE | re.IGNORECASE).match _all_zeros = re.compile('0*$').match @@ -6082,11 +6132,15 @@ def _convert_for_comparison(self, other, equality_op=False): (?Pz)? (?P\#)? (?P0)? -(?P(?!0)\d+)? -(?P,)? -(?:\.(?P0|(?!0)\d+))? +(?P\d+)? +(?P[,_])? +(?:\. + (?=[\d,_]) # lookahead for digit or separator + (?P\d+)? + (?P[,_])? +)? (?P[eEfFgGn%])? -\Z +\z """, re.VERBOSE|re.DOTALL) del re @@ -6177,6 +6231,9 @@ def _parse_format_specifier(format_spec, _localeconv=None): format_dict['grouping'] = [3, 0] format_dict['decimal_point'] = '.' + if format_dict['frac_separators'] is None: + format_dict['frac_separators'] = '' + return format_dict def _format_align(sign, body, spec): @@ -6296,6 +6353,11 @@ def _format_number(is_negative, intpart, fracpart, exp, spec): sign = _format_sign(is_negative, spec) + frac_sep = spec['frac_separators'] + if fracpart and frac_sep: + fracpart = frac_sep.join(fracpart[pos:pos + 3] + for pos in range(0, len(fracpart), 3)) + if fracpart or spec['alt']: fracpart = spec['decimal_point'] + fracpart diff --git a/Lib/_pyio.py b/Lib/_pyio.py index 48c8f770f81..116ce4f37ec 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -16,15 +16,16 @@ _setmode = None import io -from io import (__all__, SEEK_SET, SEEK_CUR, SEEK_END) +from io import (__all__, SEEK_SET, SEEK_CUR, SEEK_END, Reader, Writer) # noqa: F401 valid_seek_flags = {0, 1, 2} # Hardwired values if hasattr(os, 'SEEK_HOLE') : valid_seek_flags.add(os.SEEK_HOLE) valid_seek_flags.add(os.SEEK_DATA) -# open() uses st_blksize whenever we can -DEFAULT_BUFFER_SIZE = 8 * 1024 # bytes +# open() uses max(min(blocksize, 8 MiB), DEFAULT_BUFFER_SIZE) +# when the device block size is available. +DEFAULT_BUFFER_SIZE = 128 * 1024 # bytes # NOTE: Base classes defined here are registered with the "official" ABCs # defined in io.py. We don't use real inheritance though, because we don't want @@ -123,10 +124,10 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None, the size of a fixed-size chunk buffer. When no buffering argument is given, the default buffering policy works as follows: - * Binary files are buffered in fixed-size chunks; the size of the buffer - is chosen using a heuristic trying to determine the underlying device's - "block size" and falling back on `io.DEFAULT_BUFFER_SIZE`. - On many systems, the buffer will typically be 4096 or 8192 bytes long. + * Binary files are buffered in fixed-size chunks; the size of the buffer + is max(min(blocksize, 8 MiB), DEFAULT_BUFFER_SIZE) + when the device block size is available. + On most systems, the buffer will typically be 128 kilobytes long. * "Interactive" text files (files for which isatty() returns True) use line buffering. Other text files use the policy described above @@ -238,18 +239,11 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None, result = raw try: line_buffering = False - if buffering == 1 or buffering < 0 and raw.isatty(): + if buffering == 1 or buffering < 0 and raw._isatty_open_only(): buffering = -1 line_buffering = True if buffering < 0: - buffering = DEFAULT_BUFFER_SIZE - try: - bs = os.fstat(raw.fileno()).st_blksize - except (OSError, AttributeError): - pass - else: - if bs > 1: - buffering = bs + buffering = max(min(raw._blksize, 8192 * 1024), DEFAULT_BUFFER_SIZE) if buffering < 0: raise ValueError("invalid buffering size") if buffering == 0: @@ -941,22 +935,22 @@ def read1(self, size=-1): return self.read(size) def write(self, b): - if self.closed: - raise ValueError("write to closed file") if isinstance(b, str): raise TypeError("can't write str to binary stream") with memoryview(b) as view: + if self.closed: + raise ValueError("write to closed file") + n = view.nbytes # Size of any bytes-like object - if n == 0: - return 0 - pos = self._pos - if pos > len(self._buffer): - # Inserts null bytes between the current end of the file - # and the new write position. - padding = b'\x00' * (pos - len(self._buffer)) - self._buffer += padding - self._buffer[pos:pos + n] = b - self._pos += n + if n == 0: + return 0 + + pos = self._pos + if pos > len(self._buffer): + # Pad buffer to pos with null bytes. + self._buffer.resize(pos) + self._buffer[pos:pos + n] = view + self._pos += n return n def seek(self, pos, whence=0): @@ -1470,6 +1464,17 @@ def write(self, b): return BufferedWriter.write(self, b) +def _new_buffersize(bytes_read): + # Parallels _io/fileio.c new_buffersize + if bytes_read > 65536: + addend = bytes_read >> 3 + else: + addend = 256 + bytes_read + if addend < DEFAULT_BUFFER_SIZE: + addend = DEFAULT_BUFFER_SIZE + return bytes_read + addend + + class FileIO(RawIOBase): _fd = -1 _created = False @@ -1494,6 +1499,7 @@ def __init__(self, file, mode='r', closefd=True, opener=None): """ if self._fd >= 0: # Have to close the existing file first. + self._stat_atopen = None try: if self._closefd: os.close(self._fd) @@ -1573,18 +1579,15 @@ def __init__(self, file, mode='r', closefd=True, opener=None): os.set_inheritable(fd, False) self._closefd = closefd - fdfstat = os.fstat(fd) + self._stat_atopen = os.fstat(fd) try: - if stat.S_ISDIR(fdfstat.st_mode): + if stat.S_ISDIR(self._stat_atopen.st_mode): raise IsADirectoryError(errno.EISDIR, os.strerror(errno.EISDIR), file) except AttributeError: # Ignore the AttributeError if stat.S_ISDIR or errno.EISDIR # don't exist. pass - self._blksize = getattr(fdfstat, 'st_blksize', 0) - if self._blksize <= 1: - self._blksize = DEFAULT_BUFFER_SIZE if _setmode: # don't translate newlines (\r\n <=> \n) @@ -1601,6 +1604,7 @@ def __init__(self, file, mode='r', closefd=True, opener=None): if e.errno != errno.ESPIPE: raise except: + self._stat_atopen = None if owned_fd is not None: os.close(owned_fd) raise @@ -1629,6 +1633,17 @@ def __repr__(self): return ('<%s name=%r mode=%r closefd=%r>' % (class_name, name, self.mode, self._closefd)) + @property + def _blksize(self): + if self._stat_atopen is None: + return DEFAULT_BUFFER_SIZE + + blksize = getattr(self._stat_atopen, "st_blksize", 0) + # WASI sets blsize to 0 + if not blksize: + return DEFAULT_BUFFER_SIZE + return blksize + def _checkReadable(self): if not self._readable: raise UnsupportedOperation('File not open for reading') @@ -1640,7 +1655,13 @@ def _checkWritable(self, msg=None): def read(self, size=None): """Read at most size bytes, returned as bytes. - Only makes one system call, so less data may be returned than requested + If size is less than 0, read all bytes in the file making + multiple read calls. See ``FileIO.readall``. + + Attempts to make only one system call, retrying only per + PEP 475 (EINTR). This means less data may be returned than + requested. + In non-blocking mode, returns None if no data is available. Return an empty bytes object at EOF. """ @@ -1656,45 +1677,57 @@ def read(self, size=None): def readall(self): """Read all data from the file, returned as bytes. - In non-blocking mode, returns as much as is immediately available, - or None if no data is available. Return an empty bytes object at EOF. + Reads until either there is an error or read() returns size 0 + (indicates EOF). If the file is already at EOF, returns an + empty bytes object. + + In non-blocking mode, returns as much data as could be read + before EAGAIN. If no data is available (EAGAIN is returned + before bytes are read) returns None. """ self._checkClosed() self._checkReadable() - bufsize = DEFAULT_BUFFER_SIZE - try: - pos = os.lseek(self._fd, 0, SEEK_CUR) - end = os.fstat(self._fd).st_size - if end >= pos: - bufsize = end - pos + 1 - except OSError: - pass + if self._stat_atopen is None or self._stat_atopen.st_size <= 0: + bufsize = DEFAULT_BUFFER_SIZE + else: + # In order to detect end of file, need a read() of at least 1 + # byte which returns size 0. Oversize the buffer by 1 byte so the + # I/O can be completed with two read() calls (one for all data, one + # for EOF) without needing to resize the buffer. + bufsize = self._stat_atopen.st_size + 1 - result = bytearray() - while True: - if len(result) >= bufsize: - bufsize = len(result) - bufsize += max(bufsize, DEFAULT_BUFFER_SIZE) - n = bufsize - len(result) - try: - chunk = os.read(self._fd, n) - except BlockingIOError: - if result: - break + if self._stat_atopen.st_size > 65536: + try: + pos = os.lseek(self._fd, 0, SEEK_CUR) + if self._stat_atopen.st_size >= pos: + bufsize = self._stat_atopen.st_size - pos + 1 + except OSError: + pass + + result = bytearray(bufsize) + bytes_read = 0 + try: + while n := os.readinto(self._fd, memoryview(result)[bytes_read:]): + bytes_read += n + if bytes_read >= len(result): + result.resize(_new_buffersize(bytes_read)) + except BlockingIOError: + if not bytes_read: return None - if not chunk: # reached the end of the file - break - result += chunk + assert len(result) - bytes_read >= 1, \ + "os.readinto buffer size 0 will result in erroneous EOF / returns 0" + result.resize(bytes_read) return bytes(result) - def readinto(self, b): + def readinto(self, buffer): """Same as RawIOBase.readinto().""" - m = memoryview(b).cast('B') - data = self.read(len(m)) - n = len(data) - m[:n] = data - return n + self._checkClosed() + self._checkReadable() + try: + return os.readinto(self._fd, buffer) + except BlockingIOError: + return None def write(self, b): """Write bytes b to file, return number written. @@ -1744,6 +1777,7 @@ def truncate(self, size=None): if size is None: size = self.tell() os.ftruncate(self._fd, size) + self._stat_atopen = None return size def close(self): @@ -1753,6 +1787,7 @@ def close(self): called more than once without error. """ if not self.closed: + self._stat_atopen = None try: if self._closefd and self._fd >= 0: os.close(self._fd) @@ -1791,6 +1826,21 @@ def isatty(self): self._checkClosed() return os.isatty(self._fd) + def _isatty_open_only(self): + """Checks whether the file is a TTY using an open-only optimization. + + TTYs are always character devices. If the interpreter knows a file is + not a character device when it would call ``isatty``, can skip that + call. Inside ``open()`` there is a fresh stat result that contains that + information. Use the stat result to skip a system call. Outside of that + context TOCTOU issues (the fd could be arbitrarily modified by + surrounding code). + """ + if (self._stat_atopen is not None + and not stat.S_ISCHR(self._stat_atopen.st_mode)): + return False + return os.isatty(self._fd) + @property def closefd(self): """True if the file descriptor will be closed by close().""" @@ -2015,8 +2065,7 @@ def __init__(self, buffer, encoding=None, errors=None, newline=None, raise ValueError("invalid encoding: %r" % encoding) if not codecs.lookup(encoding)._is_text_encoding: - msg = ("%r is not a text encoding; " - "use codecs.open() to handle arbitrary codecs") + msg = "%r is not a text encoding" raise LookupError(msg % encoding) if errors is None: @@ -2524,9 +2573,12 @@ def read(self, size=None): size = size_index() decoder = self._decoder or self._get_decoder() if size < 0: + chunk = self.buffer.read() + if chunk is None: + raise BlockingIOError("Read returned None.") # Read everything. result = (self._get_decoded_chars() + - decoder.decode(self.buffer.read(), final=True)) + decoder.decode(chunk, final=True)) if self._snapshot is not None: self._set_decoded_chars('') self._snapshot = None diff --git a/Lib/_pylong.py b/Lib/_pylong.py index 4970eb3fa67..be1acd17ce3 100644 --- a/Lib/_pylong.py +++ b/Lib/_pylong.py @@ -45,10 +45,16 @@ # # and `mycache[lo]` replaces `base**lo` in the inner function. # -# While this does give minor speedups (a few percent at best), the primary -# intent is to simplify the functions using this, by eliminating the need for -# them to craft their own ad-hoc caching schemes. -def compute_powers(w, base, more_than, show=False): +# If an algorithm wants the powers of ceiling(w/2) instead of the floor, +# pass keyword argument `need_hi=True`. +# +# While this does give minor speedups (a few percent at best), the +# primary intent is to simplify the functions using this, by eliminating +# the need for them to craft their own ad-hoc caching schemes. +# +# See code near end of file for a block of code that can be enabled to +# run millions of tests. +def compute_powers(w, base, more_than, *, need_hi=False, show=False): seen = set() need = set() ws = {w} @@ -58,40 +64,70 @@ def compute_powers(w, base, more_than, show=False): continue seen.add(w) lo = w >> 1 - # only _need_ lo here; some other path may, or may not, need hi - need.add(lo) - ws.add(lo) - if w & 1: - ws.add(lo + 1) + hi = w - lo + # only _need_ one here; the other may, or may not, be needed + which = hi if need_hi else lo + need.add(which) + ws.add(which) + if lo != hi: + ws.add(w - which) + + # `need` is the set of exponents needed. To compute them all + # efficiently, possibly add other exponents to `extra`. The goal is + # to ensure that each exponent can be gotten from a smaller one via + # multiplying by the base, squaring it, or squaring and then + # multiplying by the base. + # + # If need_hi is False, this is already the case (w can always be + # gotten from w >> 1 via one of the squaring strategies). But we do + # the work anyway, just in case ;-) + # + # Note that speed is irrelevant. These loops are working on little + # ints (exponents) and go around O(log w) times. The total cost is + # insignificant compared to just one of the bigint multiplies. + cands = need.copy() + extra = set() + while cands: + w = max(cands) + cands.remove(w) + lo = w >> 1 + if lo > more_than and w-1 not in cands and lo not in cands: + extra.add(lo) + cands.add(lo) + assert need_hi or not extra d = {} - if not need: - return d - it = iter(sorted(need)) - first = next(it) - if show: - print("pow at", first) - d[first] = base ** first - for this in it: - if this - 1 in d: + for n in sorted(need | extra): + lo = n >> 1 + hi = n - lo + if n-1 in d: if show: - print("* base at", this) - d[this] = d[this - 1] * base # cheap - else: - lo = this >> 1 - hi = this - lo - assert lo in d + print("* base", end="") + result = d[n-1] * base # cheap! + elif lo in d: + # Multiplying a bigint by itself is about twice as fast + # in CPython provided it's the same object. if show: - print("square at", this) - # Multiplying a bigint by itself (same object!) is about twice - # as fast in CPython. - sq = d[lo] * d[lo] + print("square", end="") + result = d[lo] * d[lo] # same object if hi != lo: - assert hi == lo + 1 if show: - print(" and * base") - sq *= base - d[this] = sq + print(" * base", end="") + assert 2 * lo + 1 == n + result *= base + else: # rare + if show: + print("pow", end='') + result = base ** n + if show: + print(" at", n, "needed" if n in need else "extra") + d[n] = result + + assert need <= d.keys() + if excess := d.keys() - need: + assert need_hi + for n in excess: + del d[n] return d _unbounded_dec_context = decimal.getcontext().copy() @@ -211,6 +247,145 @@ def inner(a, b): return inner(0, len(s)) +# Asymptotically faster version, using the C decimal module. See +# comments at the end of the file. This uses decimal arithmetic to +# convert from base 10 to base 256. The latter is just a string of +# bytes, which CPython can convert very efficiently to a Python int. + +# log of 10 to base 256 with best-possible 53-bit precision. Obtained +# via: +# from mpmath import mp +# mp.prec = 1000 +# print(float(mp.log(10, 256)).hex()) +_LOG_10_BASE_256 = float.fromhex('0x1.a934f0979a371p-2') # about 0.415 + +# _spread is for internal testing. It maps a key to the number of times +# that condition obtained in _dec_str_to_int_inner: +# key 0 - quotient guess was right +# key 1 - quotient had to be boosted by 1, one time +# key 999 - one adjustment wasn't enough, so fell back to divmod +from collections import defaultdict +_spread = defaultdict(int) +del defaultdict + +def _dec_str_to_int_inner(s, *, GUARD=8): + # Yes, BYTELIM is "large". Large enough that CPython will usually + # use the Karatsuba _str_to_int_inner to convert the string. This + # allowed reducing the cutoff for calling _this_ function from 3.5M + # to 2M digits. We could almost certainly do even better by + # fine-tuning this and/or using a larger output base than 256. + BYTELIM = 100_000 + D = decimal.Decimal + result = bytearray() + # See notes at end of file for discussion of GUARD. + assert GUARD > 0 # if 0, `decimal` can blow up - .prec 0 not allowed + + def inner(n, w): + #assert n < D256 ** w # required, but too expensive to check + if w <= BYTELIM: + # XXX Stefan Pochmann discovered that, for 1024-bit ints, + # `int(Decimal)` took 2.5x longer than `int(str(Decimal))`. + # Worse, `int(Decimal) is still quadratic-time for much + # larger ints. So unless/until all that is repaired, the + # seemingly redundant `str(Decimal)` is crucial to speed. + result.extend(int(str(n)).to_bytes(w)) # big-endian default + return + w1 = w >> 1 + w2 = w - w1 + if 0: + # This is maximally clear, but "too slow". `decimal` + # division is asymptotically fast, but we have no way to + # tell it to reuse the high-precision reciprocal it computes + # for pow256[w2], so it has to recompute it over & over & + # over again :-( + hi, lo = divmod(n, pow256[w2][0]) + else: + p256, recip = pow256[w2] + # The integer part will have a number of digits about equal + # to the difference between the log10s of `n` and `pow256` + # (which, since these are integers, is roughly approximated + # by `.adjusted()`). That's the working precision we need, + ctx.prec = max(n.adjusted() - p256.adjusted(), 0) + GUARD + hi = +n * +recip # unary `+` chops back to ctx.prec digits + ctx.prec = decimal.MAX_PREC + hi = hi.to_integral_value() # lose the fractional digits + lo = n - hi * p256 + # Because we've been uniformly rounding down, `hi` is a + # lower bound on the correct quotient. + assert lo >= 0 + # Adjust quotient up if needed. It usually isn't. In random + # testing on inputs through 5 billion digit strings, the + # test triggered once in about 200 thousand tries. + count = 0 + if lo >= p256: + count = 1 + lo -= p256 + hi += 1 + if lo >= p256: + # Complete correction via an exact computation. I + # believe it's not possible to get here provided + # GUARD >= 3. It's tested by reducing GUARD below + # that. + count = 999 + hi2, lo = divmod(lo, p256) + hi += hi2 + _spread[count] += 1 + # The assert should always succeed, but way too slow to keep + # enabled. + #assert hi, lo == divmod(n, pow256[w2][0]) + inner(hi, w1) + del hi # at top levels, can free a lot of RAM "early" + inner(lo, w2) + + # How many base 256 digits are needed?. Mathematically, exactly + # floor(log256(int(s))) + 1. There is no cheap way to compute this. + # But we can get an upper bound, and that's necessary for our error + # analysis to make sense. int(s) < 10**len(s), so the log needed is + # < log256(10**len(s)) = len(s) * log256(10). However, using + # finite-precision floating point for this, it's possible that the + # computed value is a little less than the true value. If the true + # value is at - or a little higher than - an integer, we can get an + # off-by-1 error too low. So we add 2 instead of 1 if chopping lost + # a fraction > 0.9. + + # The "WASI" test platform can complain about `len(s)` if it's too + # large to fit in its idea of "an index-sized integer". + lenS = s.__len__() + log_ub = lenS * _LOG_10_BASE_256 + log_ub_as_int = int(log_ub) + w = log_ub_as_int + 1 + (log_ub - log_ub_as_int > 0.9) + # And what if we've plain exhausted the limits of HW floats? We + # could compute the log to any desired precision using `decimal`, + # but it's not plausible that anyone will pass a string requiring + # trillions of bytes (unless they're just trying to "break things"). + if w.bit_length() >= 46: + # "Only" had < 53 - 46 = 7 bits to spare in IEEE-754 double. + raise ValueError(f"cannot convert string of len {lenS} to int") + with decimal.localcontext(_unbounded_dec_context) as ctx: + D256 = D(256) + pow256 = compute_powers(w, D256, BYTELIM, need_hi=True) + rpow256 = compute_powers(w, 1 / D256, BYTELIM, need_hi=True) + # We're going to do inexact, chopped arithmetic, multiplying by + # an approximation to the reciprocal of 256**i. We chop to get a + # lower bound on the true integer quotient. Our approximation is + # a lower bound, the multiplication is chopped too, and + # to_integral_value() is also chopped. + ctx.traps[decimal.Inexact] = 0 + ctx.rounding = decimal.ROUND_DOWN + for k, v in pow256.items(): + # No need to save much more precision in the reciprocal than + # the power of 256 has, plus some guard digits to absorb + # most relevant rounding errors. This is highly significant: + # 1/2**i has the same number of significant decimal digits + # as 5**i, generally over twice the number in 2**i, + ctx.prec = v.adjusted() + GUARD + 1 + # The unary "+" chops the reciprocal back to that precision. + pow256[k] = v, +rpow256[k] + del rpow256 # exact reciprocals no longer needed + ctx.prec = decimal.MAX_PREC + inner(D(s), w) + return int.from_bytes(result) + def int_from_string(s): """Asymptotically fast version of PyLong_FromString(), conversion of a string of decimal digits into an 'int'.""" @@ -219,7 +394,10 @@ def int_from_string(s): # and underscores, and stripped leading whitespace. The input can still # contain underscores and have trailing whitespace. s = s.rstrip().replace('_', '') - return _str_to_int_inner(s) + func = _str_to_int_inner + if len(s) >= 2_000_000 and _decimal is not None: + func = _dec_str_to_int_inner + return func(s) def str_to_int(s): """Asymptotically fast version of decimal string to 'int' conversion.""" @@ -352,7 +530,7 @@ def int_divmod(a, b): Its time complexity is O(n**1.58), where n = #bits(a) + #bits(b). """ if b == 0: - raise ZeroDivisionError + raise ZeroDivisionError('division by zero') elif b < 0: q, r = int_divmod(-a, -b) return q, -r @@ -361,3 +539,191 @@ def int_divmod(a, b): return ~q, b + ~r else: return _divmod_pos(a, b) + + +# Notes on _dec_str_to_int_inner: +# +# Stefan Pochmann worked up a str->int function that used the decimal +# module to, in effect, convert from base 10 to base 256. This is +# "unnatural", in that it requires multiplying and dividing by large +# powers of 2, which `decimal` isn't naturally suited to. But +# `decimal`'s `*` and `/` are asymptotically superior to CPython's, so +# at _some_ point it could be expected to win. +# +# Alas, the crossover point was too high to be of much real interest. I +# (Tim) then worked on ways to replace its division with multiplication +# by a cached reciprocal approximation instead, fixing up errors +# afterwards. This reduced the crossover point significantly, +# +# I revisited the code, and found ways to improve and simplify it. The +# crossover point is at about 3.4 million digits now. +# +# About .adjusted() +# ----------------- +# Restrict to Decimal values x > 0. We don't use negative numbers in the +# code, and I don't want to have to keep typing, e.g., "absolute value". +# +# For convenience, I'll use `x.a` to mean `x.adjusted()`. x.a doesn't +# look at the digits of x, but instead returns an integer giving x's +# order of magnitude. These are equivalent: +# +# - x.a is the power-of-10 exponent of x's most significant digit. +# - x.a = the infinitely precise floor(log10(x)) +# - x can be written in this form, where f is a real with 1 <= f < 10: +# x = f * 10**x.a +# +# Observation; if x is an integer, len(str(x)) = x.a + 1. +# +# Lemma 1: (x * y).a = x.a + y.a, or one larger +# +# Proof: Write x = f * 10**x.a and y = g * 10**y.a, where f and g are in +# [1, 10). Then x*y = f*g * 10**(x.a + y.a), where 1 <= f*g < 100. If +# f*g < 10, (x*y).a is x.a+y.a. Else divide f*g by 10 to bring it back +# into [1, 10], and add 1 to the exponent to compensate. Then (x*y).a is +# x.a+y.a+1. +# +# Lemma 2: ceiling(log10(x/y)) <= x.a - y.a + 1 +# +# Proof: Express x and y as in Lemma 1. Then x/y = f/g * 10**(x.a - +# y.a), where 1/10 < f/g < 10. If 1 <= f/g, (x/y).a is x.a-y.a. Else +# multiply f/g by 10 to bring it back into [1, 10], and subtract 1 from +# the exponent to compensate. Then (x/y).a is x.a-y.a-1. So the largest +# (x/y).a can be is x.a-y.a. Since that's the floor of log10(x/y). the +# ceiling is at most 1 larger (with equality iff f/g = 1 exactly). +# +# GUARD digits +# ------------ +# We only want the integer part of divisions, so don't need to build +# the full multiplication tree. But using _just_ the number of +# digits expected in the integer part ignores too much. What's left +# out can have a very significant effect on the quotient. So we use +# GUARD additional digits. +# +# The default 8 is more than enough so no more than 1 correction step +# was ever needed for all inputs tried through 2.5 billion digits. In +# fact, I believe 3 guard digits are always enough - but the proof is +# very involved, so better safe than sorry. +# +# Short course: +# +# If prec is the decimal precision in effect, and we're rounding down, +# the result of an operation is exactly equal to the infinitely precise +# result times 1-e for some real e with 0 <= e < 10**(1-prec). In +# +# ctx.prec = max(n.adjusted() - p256.adjusted(), 0) + GUARD +# hi = +n * +recip # unary `+` chops to ctx.prec digits +# +# we have 3 visible chopped operations, but there's also a 4th: +# precomputing a truncated `recip` as part of setup. +# +# So the computed product is exactly equal to the true product times +# (1-e1)*(1-e2)*(1-e3)*(1-e4); since the e's are all very small, an +# excellent approximation to the second factor is 1-(e1+e2+e3+e4) (the +# 2nd and higher order terms in the expanded product are too tiny to +# matter). If they're all as large as possible, that's +# +# 1 - 4*10**(1-prec). This, BTW, is all bog-standard FP error analysis. +# +# That implies the computed product is within 1 of the true product +# provided prec >= log10(true_product) + 1.602. +# +# Here are telegraphic details, rephrasing the initial condition in +# equivalent ways, step by step: +# +# prod - prod * (1 - 4*10**(1-prec)) <= 1 +# prod - prod + prod * 4*10**(1-prec)) <= 1 +# prod * 4*10**(1-prec)) <= 1 +# 10**(log10(prod)) * 4*10**(1-prec)) <= 1 +# 4*10**(1-prec+log10(prod))) <= 1 +# 10**(1-prec+log10(prod))) <= 1/4 +# 1-prec+log10(prod) <= log10(1/4) = -0.602 +# -prec <= -1.602 - log10(prod) +# prec >= log10(prod) + 1.602 +# +# The true product is the same as the true ratio n/p256. By Lemma 2 +# above, n.a - p256.a + 1 is an upper bound on the ceiling of +# log10(prod). Then 2 is the ceiling of 1.602. so n.a - p256.a + 3 is an +# upper bound on the right hand side of the inequality. Any prec >= that +# will work. +# +# But since this is just a sketch of a proof ;-), the code uses the +# empirically tested 8 instead of 3. 5 digits more or less makes no +# practical difference to speed - these ints are huge. And while +# increasing GUARD above 3 may not be necessary, every increase cuts the +# percentage of cases that need a correction at all. +# +# On Computing Reciprocals +# ------------------------ +# In general, the exact reciprocals we compute have over twice as many +# significant digits as needed. 1/256**i has the same number of +# significant decimal digits as 5**i. It's a significant waste of RAM +# to store all those unneeded digits. +# +# So we cut exact reciprocals back to the least precision that can +# be needed so that the error analysis above is valid, +# +# [Note: turns out it's very significantly faster to do it this way than +# to compute 1 / 256**i directly to the desired precision, because the +# power method doesn't require division. It's also faster than computing +# (1/256)**i directly to the desired precision - no material division +# there, but `compute_powers()` is much smarter about _how_ to compute +# all the powers needed than repeated applications of `**` - that +# function invokes `**` for at most the few smallest powers needed.] +# +# The hard part is that chopping back to a shorter width occurs +# _outside_ of `inner`. We can't know then what `prec` `inner()` will +# need. We have to pick, for each value of `w2`, the largest possible +# value `prec` can become when `inner()` is working on `w2`. +# +# This is the `prec` inner() uses: +# max(n.a - p256.a, 0) + GUARD +# and what setup uses (renaming its `v` to `p256` - same thing): +# p256.a + GUARD + 1 +# +# We need that the second is always at least as large as the first, +# which is the same as requiring +# +# n.a - 2 * p256.a <= 1 +# +# What's the largest n can be? n < 255**w = 256**(w2 + (w - w2)). The +# worst case in this context is when w ix even. and then w = 2*w2, so +# n < 256**(2*w2) = (256**w2)**2 = p256**2. By Lemma 1, then, n.a +# is at most p256.a + p256.a + 1. +# +# So the most n.a - 2 * p256.a can be is +# p256.a + p256.a + 1 - 2 * p256.a = 1. QED +# +# Note: an earlier version of the code split on floor(e/2) instead of on +# the ceiling. The worst case then is odd `w`, and a more involved proof +# was needed to show that adding 4 (instead of 1) may be necessary. +# Basically because, in that case, n may be up to 256 times larger than +# p256**2. Curiously enough, by splitting on the ceiling instead, +# nothing in any proof here actually depends on the output base (256). + +# Enable for brute-force testing of compute_powers(). This takes about a +# minute, because it tries millions of cases. +if 0: + def consumer(w, limit, need_hi): + seen = set() + need = set() + def inner(w): + if w <= limit: + return + if w in seen: + return + seen.add(w) + lo = w >> 1 + hi = w - lo + need.add(hi if need_hi else lo) + inner(lo) + inner(hi) + inner(w) + exp = compute_powers(w, 1, limit, need_hi=need_hi) + assert exp.keys() == need + + from itertools import chain + for need_hi in (False, True): + for limit in (0, 1, 10, 100, 1_000, 10_000, 100_000): + for w in chain(range(1, 100_000), + (10**i for i in range(5, 30))): + consumer(w, limit, need_hi) diff --git a/Lib/_strptime.py b/Lib/_strptime.py index a07eb5f9230..fc7e369c3d1 100644 --- a/Lib/_strptime.py +++ b/Lib/_strptime.py @@ -472,7 +472,7 @@ def repl(m): if day_of_month_in_format and not year_in_format: import warnings warnings.warn("""\ -Parsing dates involving a day of month without a year specified is ambiguious +Parsing dates involving a day of month without a year specified is ambiguous and fails to parse leap day. The default behavior will change in Python 3.15 to either always raise an exception or to use a different default year (TBD). To avoid trouble, add a specific year to the input & format. @@ -783,18 +783,40 @@ def _strptime_time(data_string, format="%a %b %d %H:%M:%S %Y"): tt = _strptime(data_string, format)[0] return time.struct_time(tt[:time._STRUCT_TM_ITEMS]) -def _strptime_datetime(cls, data_string, format="%a %b %d %H:%M:%S %Y"): - """Return a class cls instance based on the input string and the +def _strptime_datetime_date(cls, data_string, format="%a %b %d %Y"): + """Return a date instance based on the input string and the + format string.""" + tt, _, _ = _strptime(data_string, format) + args = tt[:3] + return cls(*args) + +def _parse_tz(tzname, gmtoff, gmtoff_fraction): + tzdelta = datetime_timedelta(seconds=gmtoff, microseconds=gmtoff_fraction) + if tzname: + return datetime_timezone(tzdelta, tzname) + else: + return datetime_timezone(tzdelta) + +def _strptime_datetime_time(cls, data_string, format="%H:%M:%S"): + """Return a time instance based on the input string and the format string.""" tt, fraction, gmtoff_fraction = _strptime(data_string, format) tzname, gmtoff = tt[-2:] - args = tt[:6] + (fraction,) - if gmtoff is not None: - tzdelta = datetime_timedelta(seconds=gmtoff, microseconds=gmtoff_fraction) - if tzname: - tz = datetime_timezone(tzdelta, tzname) - else: - tz = datetime_timezone(tzdelta) - args += (tz,) + args = tt[3:6] + (fraction,) + if gmtoff is None: + return cls(*args) + else: + tz = _parse_tz(tzname, gmtoff, gmtoff_fraction) + return cls(*args, tz) - return cls(*args) +def _strptime_datetime_datetime(cls, data_string, format="%a %b %d %H:%M:%S %Y"): + """Return a datetime instance based on the input string and the + format string.""" + tt, fraction, gmtoff_fraction = _strptime(data_string, format) + tzname, gmtoff = tt[-2:] + args = tt[:6] + (fraction,) + if gmtoff is None: + return cls(*args) + else: + tz = _parse_tz(tzname, gmtoff, gmtoff_fraction) + return cls(*args, tz) diff --git a/Lib/_weakrefset.py b/Lib/_weakrefset.py index 489eec714e0..d1c7fcaeec9 100644 --- a/Lib/_weakrefset.py +++ b/Lib/_weakrefset.py @@ -8,69 +8,29 @@ __all__ = ['WeakSet'] -class _IterationGuard: - # This context manager registers itself in the current iterators of the - # weak container, such as to delay all removals until the context manager - # exits. - # This technique should be relatively thread-safe (since sets are). - - def __init__(self, weakcontainer): - # Don't create cycles - self.weakcontainer = ref(weakcontainer) - - def __enter__(self): - w = self.weakcontainer() - if w is not None: - w._iterating.add(self) - return self - - def __exit__(self, e, t, b): - w = self.weakcontainer() - if w is not None: - s = w._iterating - s.remove(self) - if not s: - w._commit_removals() - - class WeakSet: def __init__(self, data=None): self.data = set() + def _remove(item, selfref=ref(self)): self = selfref() if self is not None: - if self._iterating: - self._pending_removals.append(item) - else: - self.data.discard(item) + self.data.discard(item) + self._remove = _remove - # A list of keys to be removed - self._pending_removals = [] - self._iterating = set() if data is not None: self.update(data) - def _commit_removals(self): - pop = self._pending_removals.pop - discard = self.data.discard - while True: - try: - item = pop() - except IndexError: - return - discard(item) - def __iter__(self): - with _IterationGuard(self): - for itemref in self.data: - item = itemref() - if item is not None: - # Caveat: the iterator will keep a strong reference to - # `item` until it is resumed or closed. - yield item + for itemref in self.data.copy(): + item = itemref() + if item is not None: + # Caveat: the iterator will keep a strong reference to + # `item` until it is resumed or closed. + yield item def __len__(self): - return len(self.data) - len(self._pending_removals) + return len(self.data) def __contains__(self, item): try: @@ -83,21 +43,15 @@ def __reduce__(self): return self.__class__, (list(self),), self.__getstate__() def add(self, item): - if self._pending_removals: - self._commit_removals() self.data.add(ref(item, self._remove)) def clear(self): - if self._pending_removals: - self._commit_removals() self.data.clear() def copy(self): return self.__class__(self) def pop(self): - if self._pending_removals: - self._commit_removals() while True: try: itemref = self.data.pop() @@ -108,18 +62,12 @@ def pop(self): return item def remove(self, item): - if self._pending_removals: - self._commit_removals() self.data.remove(ref(item)) def discard(self, item): - if self._pending_removals: - self._commit_removals() self.data.discard(ref(item)) def update(self, other): - if self._pending_removals: - self._commit_removals() for element in other: self.add(element) @@ -136,8 +84,6 @@ def difference(self, other): def difference_update(self, other): self.__isub__(other) def __isub__(self, other): - if self._pending_removals: - self._commit_removals() if self is other: self.data.clear() else: @@ -151,8 +97,6 @@ def intersection(self, other): def intersection_update(self, other): self.__iand__(other) def __iand__(self, other): - if self._pending_removals: - self._commit_removals() self.data.intersection_update(ref(item) for item in other) return self @@ -184,8 +128,6 @@ def symmetric_difference(self, other): def symmetric_difference_update(self, other): self.__ixor__(other) def __ixor__(self, other): - if self._pending_removals: - self._commit_removals() if self is other: self.data.clear() else: diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index a5788cdbfae..832d160de7f 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -279,7 +279,13 @@ def __eq__(self, other): # because dictionaries are not hashable. and self.__globals__ is other.__globals__ and self.__forward_is_class__ == other.__forward_is_class__ - and self.__cell__ == other.__cell__ + # Two separate cells are always considered unequal in forward refs. + and ( + {name: id(cell) for name, cell in self.__cell__.items()} + == {name: id(cell) for name, cell in other.__cell__.items()} + if isinstance(self.__cell__, dict) and isinstance(other.__cell__, dict) + else self.__cell__ is other.__cell__ + ) and self.__owner__ == other.__owner__ and ( (tuple(sorted(self.__extra_names__.items())) if self.__extra_names__ else None) == @@ -293,7 +299,10 @@ def __hash__(self): self.__forward_module__, id(self.__globals__), # dictionaries are not hashable, so hash by identity self.__forward_is_class__, - tuple(sorted(self.__cell__.items())) if isinstance(self.__cell__, dict) else self.__cell__, + ( # cells are not hashable as well + tuple(sorted([(name, id(cell)) for name, cell in self.__cell__.items()])) + if isinstance(self.__cell__, dict) else id(self.__cell__), + ), self.__owner__, tuple(sorted(self.__extra_names__.items())) if self.__extra_names__ else None, )) @@ -1096,7 +1105,7 @@ def _rewrite_star_unpack(arg): """If the given argument annotation expression is a star unpack e.g. `'*Ts'` rewrite it to a valid expression. """ - if arg.startswith("*"): + if arg.lstrip().startswith("*"): return f"({arg},)[0]" # E.g. (*Ts,)[0] or (*tuple[int, int],)[0] else: return arg diff --git a/Lib/argparse.py b/Lib/argparse.py index 88c1f5a7ef3..1d7d34f9924 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -337,27 +337,17 @@ def _format_usage(self, usage, actions, groups, prefix): elif usage is None: prog = '%(prog)s' % dict(prog=self._prog) - # split optionals from positionals - optionals = [] - positionals = [] - for action in actions: - if action.option_strings: - optionals.append(action) - else: - positionals.append(action) - + parts, pos_start = self._get_actions_usage_parts(actions, groups) # build full usage string - format = self._format_actions_usage - action_usage = format(optionals + positionals, groups) - usage = ' '.join([s for s in [prog, action_usage] if s]) + usage = ' '.join(filter(None, [prog, *parts])) # wrap the usage parts if it's too long text_width = self._width - self._current_indent if len(prefix) + len(self._decolor(usage)) > text_width: # break usage into wrappable parts - opt_parts = self._get_actions_usage_parts(optionals, groups) - pos_parts = self._get_actions_usage_parts(positionals, groups) + opt_parts = parts[:pos_start] + pos_parts = parts[pos_start:] # helper for wrapping lines def get_lines(parts, indent, prefix=None): @@ -414,110 +404,114 @@ def get_lines(parts, indent, prefix=None): # prefix with 'usage:' return f'{t.usage}{prefix}{t.reset}{usage}\n\n' - def _format_actions_usage(self, actions, groups): - return ' '.join(self._get_actions_usage_parts(actions, groups)) - def _is_long_option(self, string): return len(string) > 2 def _get_actions_usage_parts(self, actions, groups): - # find group indices and identify actions in groups - group_actions = set() - inserts = {} - for group in groups: - if not group._group_actions: - raise ValueError(f'empty group {group}') + """Get usage parts with split index for optionals/positionals. - if all(action.help is SUPPRESS for action in group._group_actions): - continue - - try: - start = min(actions.index(item) for item in group._group_actions) - except ValueError: - continue - else: - end = start + len(group._group_actions) - if set(actions[start:end]) == set(group._group_actions): - group_actions.update(group._group_actions) - inserts[start, end] = group + Returns (parts, pos_start) where pos_start is the index in parts + where positionals begin. + This preserves mutually exclusive group formatting across the + optionals/positionals boundary (gh-75949). + """ + actions = [action for action in actions if action.help is not SUPPRESS] + # group actions by mutually exclusive groups + action_groups = dict.fromkeys(actions) + for group in groups: + for action in group._group_actions: + if action in action_groups: + action_groups[action] = group + # positional arguments keep their position + positionals = [] + for action in actions: + if not action.option_strings: + group = action_groups.pop(action) + if group: + group_actions = [ + action2 for action2 in group._group_actions + if action2.option_strings and + action_groups.pop(action2, None) + ] + [action] + positionals.append((group.required, group_actions)) + else: + positionals.append((None, [action])) + # the remaining optional arguments are sorted by the position of + # the first option in the group + optionals = [] + for action in actions: + if action.option_strings and action in action_groups: + group = action_groups.pop(action) + if group: + group_actions = [action] + [ + action2 for action2 in group._group_actions + if action2.option_strings and + action_groups.pop(action2, None) + ] + optionals.append((group.required, group_actions)) + else: + optionals.append((None, [action])) # collect all actions format strings parts = [] t = self._theme - for action in actions: - - # suppressed arguments are marked with None - if action.help is SUPPRESS: - part = None - - # produce all arg strings - elif not action.option_strings: - default = self._get_default_metavar_for_positional(action) - part = ( - t.summary_action - + self._format_args(action, default) - + t.reset - ) - - # if it's in a group, strip the outer [] - if action in group_actions: - if part[0] == '[' and part[-1] == ']': - part = part[1:-1] - - # produce the first way to invoke the option in brackets - else: - option_string = action.option_strings[0] - if self._is_long_option(option_string): - option_color = t.summary_long_option + pos_start = None + for i, (required, group) in enumerate(optionals + positionals): + start = len(parts) + if i == len(optionals): + pos_start = start + in_group = len(group) > 1 + for action in group: + # produce all arg strings + if not action.option_strings: + default = self._get_default_metavar_for_positional(action) + part = self._format_args(action, default) + # if it's in a group, strip the outer [] + if in_group: + if part[0] == '[' and part[-1] == ']': + part = part[1:-1] + part = t.summary_action + part + t.reset + + # produce the first way to invoke the option in brackets else: - option_color = t.summary_short_option + option_string = action.option_strings[0] + if self._is_long_option(option_string): + option_color = t.summary_long_option + else: + option_color = t.summary_short_option - # if the Optional doesn't take a value, format is: - # -s or --long - if action.nargs == 0: - part = action.format_usage() - part = f"{option_color}{part}{t.reset}" + # if the Optional doesn't take a value, format is: + # -s or --long + if action.nargs == 0: + part = action.format_usage() + part = f"{option_color}{part}{t.reset}" - # if the Optional takes a value, format is: - # -s ARGS or --long ARGS - else: - default = self._get_default_metavar_for_optional(action) - args_string = self._format_args(action, default) - part = ( - f"{option_color}{option_string} " - f"{t.summary_label}{args_string}{t.reset}" - ) - - # make it look optional if it's not required or in a group - if not action.required and action not in group_actions: - part = '[%s]' % part - - # add the action string to the list - parts.append(part) - - # group mutually exclusive actions - inserted_separators_indices = set() - for start, end in sorted(inserts, reverse=True): - group = inserts[start, end] - group_parts = [item for item in parts[start:end] if item is not None] - group_size = len(group_parts) - if group.required: - open, close = "()" if group_size > 1 else ("", "") - else: - open, close = "[]" - group_parts[0] = open + group_parts[0] - group_parts[-1] = group_parts[-1] + close - for i, part in enumerate(group_parts[:-1], start=start): - # insert a separator if not already done in a nested group - if i not in inserted_separators_indices: - parts[i] = part + ' |' - inserted_separators_indices.add(i) - parts[start + group_size - 1] = group_parts[-1] - for i in range(start + group_size, end): - parts[i] = None - - # return the usage parts - return [item for item in parts if item is not None] + # if the Optional takes a value, format is: + # -s ARGS or --long ARGS + else: + default = self._get_default_metavar_for_optional(action) + args_string = self._format_args(action, default) + part = ( + f"{option_color}{option_string} " + f"{t.summary_label}{args_string}{t.reset}" + ) + + # make it look optional if it's not required or in a group + if not (action.required or required or in_group): + part = '[%s]' % part + + # add the action string to the list + parts.append(part) + + if in_group: + parts[start] = ('(' if required else '[') + parts[start] + for i in range(start, len(parts) - 1): + parts[i] += ' |' + parts[-1] += ')' if required else ']' + + if pos_start is None: + pos_start = len(parts) + return parts, pos_start def _format_text(self, text): if '%(prog)' in text: @@ -745,11 +739,14 @@ def _get_help_string(self, action): if help is None: help = '' - if '%(default)' not in help: - if action.default is not SUPPRESS: - defaulting_nargs = [OPTIONAL, ZERO_OR_MORE] - if action.option_strings or action.nargs in defaulting_nargs: - help += _(' (default: %(default)s)') + if ( + '%(default)' not in help + and action.default is not SUPPRESS + and not action.required + ): + defaulting_nargs = (OPTIONAL, ZERO_OR_MORE) + if action.option_strings or action.nargs in defaulting_nargs: + help += _(' (default: %(default)s)') return help @@ -1552,8 +1549,8 @@ def add_argument(self, *args, **kwargs): f'instance of it must be passed') # raise an error if the metavar does not match the type - if hasattr(self, "_get_formatter"): - formatter = self._get_formatter() + if hasattr(self, "_get_validation_formatter"): + formatter = self._get_validation_formatter() try: formatter._format_args(action, None) except TypeError: @@ -1741,8 +1738,8 @@ def _handle_conflict_resolve(self, action, conflicting_actions): action.container._remove_action(action) def _check_help(self, action): - if action.help and hasattr(self, "_get_formatter"): - formatter = self._get_formatter() + if action.help and hasattr(self, "_get_validation_formatter"): + formatter = self._get_validation_formatter() try: formatter._expand_help(action) except (ValueError, TypeError, KeyError) as exc: @@ -1897,6 +1894,9 @@ def __init__(self, self.suggest_on_error = suggest_on_error self.color = color + # Cached formatter for validation (avoids repeated _set_color calls) + self._cached_formatter = None + add_group = self.add_argument_group self._positionals = add_group(_('positional arguments')) self._optionals = add_group(_('options')) @@ -2728,6 +2728,13 @@ def _get_formatter(self): formatter._set_color(self.color) return formatter + def _get_validation_formatter(self): + # Return cached formatter for read-only validation operations + # (_expand_help and _format_args). Avoids repeated slow _set_color calls. + if self._cached_formatter is None: + self._cached_formatter = self._get_formatter() + return self._cached_formatter + # ===================== # Help-printing methods # ===================== diff --git a/Lib/ast.py b/Lib/ast.py index 37b20206b8a..2f11683ecf7 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -20,11 +20,7 @@ :copyright: Copyright 2008 by Armin Ronacher. :license: Python License. """ -import sys -import re from _ast import * -from contextlib import contextmanager, nullcontext -from enum import IntEnum, auto, _simple_enum def parse(source, filename='', mode='exec', *, @@ -319,12 +315,18 @@ def get_docstring(node, clean=True): return text -_line_pattern = re.compile(r"(.*?(?:\r\n|\n|\r|$))") +_line_pattern = None def _splitlines_no_ff(source, maxlines=None): """Split a string into lines ignoring form feed and other chars. This mimics how the Python parser splits source code. """ + global _line_pattern + if _line_pattern is None: + # lazily computed to speedup import time of `ast` + import re + _line_pattern = re.compile(r"(.*?(?:\r\n|\n|\r|$))") + lines = [] for lineno, match in enumerate(_line_pattern.finditer(source), 1): if maxlines is not None and lineno > maxlines: @@ -395,6 +397,88 @@ def walk(node): yield node +def compare( + a, + b, + /, + *, + compare_attributes=False, +): + """Recursively compares two ASTs. + + compare_attributes affects whether AST attributes are considered + in the comparison. If compare_attributes is False (default), then + attributes are ignored. Otherwise they must all be equal. This + option is useful to check whether the ASTs are structurally equal but + might differ in whitespace or similar details. + """ + + sentinel = object() # handle the possibility of a missing attribute/field + + def _compare(a, b): + # Compare two fields on an AST object, which may themselves be + # AST objects, lists of AST objects, or primitive ASDL types + # like identifiers and constants. + if isinstance(a, AST): + return compare( + a, + b, + compare_attributes=compare_attributes, + ) + elif isinstance(a, list): + # If a field is repeated, then both objects will represent + # the value as a list. + if len(a) != len(b): + return False + for a_item, b_item in zip(a, b): + if not _compare(a_item, b_item): + return False + else: + return True + else: + return type(a) is type(b) and a == b + + def _compare_fields(a, b): + if a._fields != b._fields: + return False + for field in a._fields: + a_field = getattr(a, field, sentinel) + b_field = getattr(b, field, sentinel) + if a_field is sentinel and b_field is sentinel: + # both nodes are missing a field at runtime + continue + if a_field is sentinel or b_field is sentinel: + # one of the node is missing a field + return False + if not _compare(a_field, b_field): + return False + else: + return True + + def _compare_attributes(a, b): + if a._attributes != b._attributes: + return False + # Attributes are always ints. + for attr in a._attributes: + a_attr = getattr(a, attr, sentinel) + b_attr = getattr(b, attr, sentinel) + if a_attr is sentinel and b_attr is sentinel: + # both nodes are missing an attribute at runtime + continue + if a_attr != b_attr: + return False + else: + return True + + if type(a) is not type(b): + return False + if not _compare_fields(a, b): + return False + if compare_attributes and not _compare_attributes(a, b): + return False + return True + + class NodeVisitor(object): """ A node visitor base class that walks the abstract syntax tree and calls a @@ -431,27 +515,6 @@ def generic_visit(self, node): elif isinstance(value, AST): self.visit(value) - def visit_Constant(self, node): - value = node.value - type_name = _const_node_type_names.get(type(value)) - if type_name is None: - for cls, name in _const_node_type_names.items(): - if isinstance(value, cls): - type_name = name - break - if type_name is not None: - method = 'visit_' + type_name - try: - visitor = getattr(self, method) - except AttributeError: - pass - else: - import warnings - warnings.warn(f"{method} is deprecated; add visit_Constant", - DeprecationWarning, 2) - return visitor(node) - return self.generic_visit(node) - class NodeTransformer(NodeVisitor): """ @@ -511,151 +574,6 @@ def generic_visit(self, node): setattr(node, field, new_node) return node - -_DEPRECATED_VALUE_ALIAS_MESSAGE = ( - "{name} is deprecated and will be removed in Python {remove}; use value instead" -) -_DEPRECATED_CLASS_MESSAGE = ( - "{name} is deprecated and will be removed in Python {remove}; " - "use ast.Constant instead" -) - - -# If the ast module is loaded more than once, only add deprecated methods once -if not hasattr(Constant, 'n'): - # The following code is for backward compatibility. - # It will be removed in future. - - def _n_getter(self): - """Deprecated. Use value instead.""" - import warnings - warnings._deprecated( - "Attribute n", message=_DEPRECATED_VALUE_ALIAS_MESSAGE, remove=(3, 14) - ) - return self.value - - def _n_setter(self, value): - import warnings - warnings._deprecated( - "Attribute n", message=_DEPRECATED_VALUE_ALIAS_MESSAGE, remove=(3, 14) - ) - self.value = value - - def _s_getter(self): - """Deprecated. Use value instead.""" - import warnings - warnings._deprecated( - "Attribute s", message=_DEPRECATED_VALUE_ALIAS_MESSAGE, remove=(3, 14) - ) - return self.value - - def _s_setter(self, value): - import warnings - warnings._deprecated( - "Attribute s", message=_DEPRECATED_VALUE_ALIAS_MESSAGE, remove=(3, 14) - ) - self.value = value - - Constant.n = property(_n_getter, _n_setter) - Constant.s = property(_s_getter, _s_setter) - -class _ABC(type): - - def __init__(cls, *args): - cls.__doc__ = """Deprecated AST node class. Use ast.Constant instead""" - - def __instancecheck__(cls, inst): - if cls in _const_types: - import warnings - warnings._deprecated( - f"ast.{cls.__qualname__}", - message=_DEPRECATED_CLASS_MESSAGE, - remove=(3, 14) - ) - if not isinstance(inst, Constant): - return False - if cls in _const_types: - try: - value = inst.value - except AttributeError: - return False - else: - return ( - isinstance(value, _const_types[cls]) and - not isinstance(value, _const_types_not.get(cls, ())) - ) - return type.__instancecheck__(cls, inst) - -def _new(cls, *args, **kwargs): - for key in kwargs: - if key not in cls._fields: - # arbitrary keyword arguments are accepted - continue - pos = cls._fields.index(key) - if pos < len(args): - raise TypeError(f"{cls.__name__} got multiple values for argument {key!r}") - if cls in _const_types: - import warnings - warnings._deprecated( - f"ast.{cls.__qualname__}", message=_DEPRECATED_CLASS_MESSAGE, remove=(3, 14) - ) - return Constant(*args, **kwargs) - return Constant.__new__(cls, *args, **kwargs) - -class Num(Constant, metaclass=_ABC): - _fields = ('n',) - __new__ = _new - -class Str(Constant, metaclass=_ABC): - _fields = ('s',) - __new__ = _new - -class Bytes(Constant, metaclass=_ABC): - _fields = ('s',) - __new__ = _new - -class NameConstant(Constant, metaclass=_ABC): - __new__ = _new - -class Ellipsis(Constant, metaclass=_ABC): - _fields = () - - def __new__(cls, *args, **kwargs): - if cls is _ast_Ellipsis: - import warnings - warnings._deprecated( - "ast.Ellipsis", message=_DEPRECATED_CLASS_MESSAGE, remove=(3, 14) - ) - return Constant(..., *args, **kwargs) - return Constant.__new__(cls, *args, **kwargs) - -# Keep another reference to Ellipsis in the global namespace -# so it can be referenced in Ellipsis.__new__ -# (The original "Ellipsis" name is removed from the global namespace later on) -_ast_Ellipsis = Ellipsis - -_const_types = { - Num: (int, float, complex), - Str: (str,), - Bytes: (bytes,), - NameConstant: (type(None), bool), - Ellipsis: (type(...),), -} -_const_types_not = { - Num: (bool,), -} - -_const_node_type_names = { - bool: 'NameConstant', # should be before int - type(None): 'NameConstant', - int: 'Num', - float: 'Num', - complex: 'Num', - str: 'Str', - bytes: 'Bytes', - type(...): 'Ellipsis', -} - class slice(AST): """Deprecated AST node class.""" @@ -696,1147 +614,21 @@ class Param(expr_context): """Deprecated AST node class. Unused in Python 3.""" -# Large float and imaginary literals get turned into infinities in the AST. -# We unparse those infinities to INFSTR. -_INFSTR = "1e" + repr(sys.float_info.max_10_exp + 1) - -@_simple_enum(IntEnum) -class _Precedence: - """Precedence table that originated from python grammar.""" - - NAMED_EXPR = auto() # := - TUPLE = auto() # , - YIELD = auto() # 'yield', 'yield from' - TEST = auto() # 'if'-'else', 'lambda' - OR = auto() # 'or' - AND = auto() # 'and' - NOT = auto() # 'not' - CMP = auto() # '<', '>', '==', '>=', '<=', '!=', - # 'in', 'not in', 'is', 'is not' - EXPR = auto() - BOR = EXPR # '|' - BXOR = auto() # '^' - BAND = auto() # '&' - SHIFT = auto() # '<<', '>>' - ARITH = auto() # '+', '-' - TERM = auto() # '*', '@', '/', '%', '//' - FACTOR = auto() # unary '+', '-', '~' - POWER = auto() # '**' - AWAIT = auto() # 'await' - ATOM = auto() - - def next(self): - try: - return self.__class__(self + 1) - except ValueError: - return self - - -_SINGLE_QUOTES = ("'", '"') -_MULTI_QUOTES = ('"""', "'''") -_ALL_QUOTES = (*_SINGLE_QUOTES, *_MULTI_QUOTES) - -class _Unparser(NodeVisitor): - """Methods in this class recursively traverse an AST and - output source code for the abstract syntax; original formatting - is disregarded.""" - - def __init__(self): - self._source = [] - self._precedences = {} - self._type_ignores = {} - self._indent = 0 - self._in_try_star = False - - def interleave(self, inter, f, seq): - """Call f on each item in seq, calling inter() in between.""" - seq = iter(seq) - try: - f(next(seq)) - except StopIteration: - pass - else: - for x in seq: - inter() - f(x) - - def items_view(self, traverser, items): - """Traverse and separate the given *items* with a comma and append it to - the buffer. If *items* is a single item sequence, a trailing comma - will be added.""" - if len(items) == 1: - traverser(items[0]) - self.write(",") - else: - self.interleave(lambda: self.write(", "), traverser, items) - - def maybe_newline(self): - """Adds a newline if it isn't the start of generated source""" - if self._source: - self.write("\n") - - def fill(self, text=""): - """Indent a piece of text and append it, according to the current - indentation level""" - self.maybe_newline() - self.write(" " * self._indent + text) - - def write(self, *text): - """Add new source parts""" - self._source.extend(text) - - @contextmanager - def buffered(self, buffer = None): - if buffer is None: - buffer = [] - - original_source = self._source - self._source = buffer - yield buffer - self._source = original_source - - @contextmanager - def block(self, *, extra = None): - """A context manager for preparing the source for blocks. It adds - the character':', increases the indentation on enter and decreases - the indentation on exit. If *extra* is given, it will be directly - appended after the colon character. - """ - self.write(":") - if extra: - self.write(extra) - self._indent += 1 - yield - self._indent -= 1 - - @contextmanager - def delimit(self, start, end): - """A context manager for preparing the source for expressions. It adds - *start* to the buffer and enters, after exit it adds *end*.""" - - self.write(start) - yield - self.write(end) - - def delimit_if(self, start, end, condition): - if condition: - return self.delimit(start, end) - else: - return nullcontext() - - def require_parens(self, precedence, node): - """Shortcut to adding precedence related parens""" - return self.delimit_if("(", ")", self.get_precedence(node) > precedence) - - def get_precedence(self, node): - return self._precedences.get(node, _Precedence.TEST) - - def set_precedence(self, precedence, *nodes): - for node in nodes: - self._precedences[node] = precedence - - def get_raw_docstring(self, node): - """If a docstring node is found in the body of the *node* parameter, - return that docstring node, None otherwise. - - Logic mirrored from ``_PyAST_GetDocString``.""" - if not isinstance( - node, (AsyncFunctionDef, FunctionDef, ClassDef, Module) - ) or len(node.body) < 1: - return None - node = node.body[0] - if not isinstance(node, Expr): - return None - node = node.value - if isinstance(node, Constant) and isinstance(node.value, str): - return node - - def get_type_comment(self, node): - comment = self._type_ignores.get(node.lineno) or node.type_comment - if comment is not None: - return f" # type: {comment}" - - def traverse(self, node): - if isinstance(node, list): - for item in node: - self.traverse(item) - else: - super().visit(node) - - # Note: as visit() resets the output text, do NOT rely on - # NodeVisitor.generic_visit to handle any nodes (as it calls back in to - # the subclass visit() method, which resets self._source to an empty list) - def visit(self, node): - """Outputs a source code string that, if converted back to an ast - (using ast.parse) will generate an AST equivalent to *node*""" - self._source = [] - self.traverse(node) - return "".join(self._source) - - def _write_docstring_and_traverse_body(self, node): - if (docstring := self.get_raw_docstring(node)): - self._write_docstring(docstring) - self.traverse(node.body[1:]) - else: - self.traverse(node.body) - - def visit_Module(self, node): - self._type_ignores = { - ignore.lineno: f"ignore{ignore.tag}" - for ignore in node.type_ignores - } - self._write_docstring_and_traverse_body(node) - self._type_ignores.clear() - - def visit_FunctionType(self, node): - with self.delimit("(", ")"): - self.interleave( - lambda: self.write(", "), self.traverse, node.argtypes - ) - - self.write(" -> ") - self.traverse(node.returns) - - def visit_Expr(self, node): - self.fill() - self.set_precedence(_Precedence.YIELD, node.value) - self.traverse(node.value) - - def visit_NamedExpr(self, node): - with self.require_parens(_Precedence.NAMED_EXPR, node): - self.set_precedence(_Precedence.ATOM, node.target, node.value) - self.traverse(node.target) - self.write(" := ") - self.traverse(node.value) - - def visit_Import(self, node): - self.fill("import ") - self.interleave(lambda: self.write(", "), self.traverse, node.names) - - def visit_ImportFrom(self, node): - self.fill("from ") - self.write("." * (node.level or 0)) - if node.module: - self.write(node.module) - self.write(" import ") - self.interleave(lambda: self.write(", "), self.traverse, node.names) - - def visit_Assign(self, node): - self.fill() - for target in node.targets: - self.set_precedence(_Precedence.TUPLE, target) - self.traverse(target) - self.write(" = ") - self.traverse(node.value) - if type_comment := self.get_type_comment(node): - self.write(type_comment) - - def visit_AugAssign(self, node): - self.fill() - self.traverse(node.target) - self.write(" " + self.binop[node.op.__class__.__name__] + "= ") - self.traverse(node.value) - - def visit_AnnAssign(self, node): - self.fill() - with self.delimit_if("(", ")", not node.simple and isinstance(node.target, Name)): - self.traverse(node.target) - self.write(": ") - self.traverse(node.annotation) - if node.value: - self.write(" = ") - self.traverse(node.value) - - def visit_Return(self, node): - self.fill("return") - if node.value: - self.write(" ") - self.traverse(node.value) - - def visit_Pass(self, node): - self.fill("pass") - - def visit_Break(self, node): - self.fill("break") - - def visit_Continue(self, node): - self.fill("continue") - - def visit_Delete(self, node): - self.fill("del ") - self.interleave(lambda: self.write(", "), self.traverse, node.targets) - - def visit_Assert(self, node): - self.fill("assert ") - self.traverse(node.test) - if node.msg: - self.write(", ") - self.traverse(node.msg) - - def visit_Global(self, node): - self.fill("global ") - self.interleave(lambda: self.write(", "), self.write, node.names) - - def visit_Nonlocal(self, node): - self.fill("nonlocal ") - self.interleave(lambda: self.write(", "), self.write, node.names) - - def visit_Await(self, node): - with self.require_parens(_Precedence.AWAIT, node): - self.write("await") - if node.value: - self.write(" ") - self.set_precedence(_Precedence.ATOM, node.value) - self.traverse(node.value) - - def visit_Yield(self, node): - with self.require_parens(_Precedence.YIELD, node): - self.write("yield") - if node.value: - self.write(" ") - self.set_precedence(_Precedence.ATOM, node.value) - self.traverse(node.value) - - def visit_YieldFrom(self, node): - with self.require_parens(_Precedence.YIELD, node): - self.write("yield from ") - if not node.value: - raise ValueError("Node can't be used without a value attribute.") - self.set_precedence(_Precedence.ATOM, node.value) - self.traverse(node.value) - - def visit_Raise(self, node): - self.fill("raise") - if not node.exc: - if node.cause: - raise ValueError(f"Node can't use cause without an exception.") - return - self.write(" ") - self.traverse(node.exc) - if node.cause: - self.write(" from ") - self.traverse(node.cause) - - def do_visit_try(self, node): - self.fill("try") - with self.block(): - self.traverse(node.body) - for ex in node.handlers: - self.traverse(ex) - if node.orelse: - self.fill("else") - with self.block(): - self.traverse(node.orelse) - if node.finalbody: - self.fill("finally") - with self.block(): - self.traverse(node.finalbody) - - def visit_Try(self, node): - prev_in_try_star = self._in_try_star - try: - self._in_try_star = False - self.do_visit_try(node) - finally: - self._in_try_star = prev_in_try_star - - def visit_TryStar(self, node): - prev_in_try_star = self._in_try_star - try: - self._in_try_star = True - self.do_visit_try(node) - finally: - self._in_try_star = prev_in_try_star - - def visit_ExceptHandler(self, node): - self.fill("except*" if self._in_try_star else "except") - if node.type: - self.write(" ") - self.traverse(node.type) - if node.name: - self.write(" as ") - self.write(node.name) - with self.block(): - self.traverse(node.body) - - def visit_ClassDef(self, node): - self.maybe_newline() - for deco in node.decorator_list: - self.fill("@") - self.traverse(deco) - self.fill("class " + node.name) - if hasattr(node, "type_params"): - self._type_params_helper(node.type_params) - with self.delimit_if("(", ")", condition = node.bases or node.keywords): - comma = False - for e in node.bases: - if comma: - self.write(", ") - else: - comma = True - self.traverse(e) - for e in node.keywords: - if comma: - self.write(", ") - else: - comma = True - self.traverse(e) - - with self.block(): - self._write_docstring_and_traverse_body(node) - - def visit_FunctionDef(self, node): - self._function_helper(node, "def") - - def visit_AsyncFunctionDef(self, node): - self._function_helper(node, "async def") - - def _function_helper(self, node, fill_suffix): - self.maybe_newline() - for deco in node.decorator_list: - self.fill("@") - self.traverse(deco) - def_str = fill_suffix + " " + node.name - self.fill(def_str) - if hasattr(node, "type_params"): - self._type_params_helper(node.type_params) - with self.delimit("(", ")"): - self.traverse(node.args) - if node.returns: - self.write(" -> ") - self.traverse(node.returns) - with self.block(extra=self.get_type_comment(node)): - self._write_docstring_and_traverse_body(node) - - def _type_params_helper(self, type_params): - if type_params is not None and len(type_params) > 0: - with self.delimit("[", "]"): - self.interleave(lambda: self.write(", "), self.traverse, type_params) - - def visit_TypeVar(self, node): - self.write(node.name) - if node.bound: - self.write(": ") - self.traverse(node.bound) - if node.default_value: - self.write(" = ") - self.traverse(node.default_value) - - def visit_TypeVarTuple(self, node): - self.write("*" + node.name) - if node.default_value: - self.write(" = ") - self.traverse(node.default_value) - - def visit_ParamSpec(self, node): - self.write("**" + node.name) - if node.default_value: - self.write(" = ") - self.traverse(node.default_value) - - def visit_TypeAlias(self, node): - self.fill("type ") - self.traverse(node.name) - self._type_params_helper(node.type_params) - self.write(" = ") - self.traverse(node.value) - - def visit_For(self, node): - self._for_helper("for ", node) - - def visit_AsyncFor(self, node): - self._for_helper("async for ", node) - - def _for_helper(self, fill, node): - self.fill(fill) - self.set_precedence(_Precedence.TUPLE, node.target) - self.traverse(node.target) - self.write(" in ") - self.traverse(node.iter) - with self.block(extra=self.get_type_comment(node)): - self.traverse(node.body) - if node.orelse: - self.fill("else") - with self.block(): - self.traverse(node.orelse) - - def visit_If(self, node): - self.fill("if ") - self.traverse(node.test) - with self.block(): - self.traverse(node.body) - # collapse nested ifs into equivalent elifs. - while node.orelse and len(node.orelse) == 1 and isinstance(node.orelse[0], If): - node = node.orelse[0] - self.fill("elif ") - self.traverse(node.test) - with self.block(): - self.traverse(node.body) - # final else - if node.orelse: - self.fill("else") - with self.block(): - self.traverse(node.orelse) - - def visit_While(self, node): - self.fill("while ") - self.traverse(node.test) - with self.block(): - self.traverse(node.body) - if node.orelse: - self.fill("else") - with self.block(): - self.traverse(node.orelse) - - def visit_With(self, node): - self.fill("with ") - self.interleave(lambda: self.write(", "), self.traverse, node.items) - with self.block(extra=self.get_type_comment(node)): - self.traverse(node.body) - - def visit_AsyncWith(self, node): - self.fill("async with ") - self.interleave(lambda: self.write(", "), self.traverse, node.items) - with self.block(extra=self.get_type_comment(node)): - self.traverse(node.body) - - def _str_literal_helper( - self, string, *, quote_types=_ALL_QUOTES, escape_special_whitespace=False - ): - """Helper for writing string literals, minimizing escapes. - Returns the tuple (string literal to write, possible quote types). - """ - def escape_char(c): - # \n and \t are non-printable, but we only escape them if - # escape_special_whitespace is True - if not escape_special_whitespace and c in "\n\t": - return c - # Always escape backslashes and other non-printable characters - if c == "\\" or not c.isprintable(): - return c.encode("unicode_escape").decode("ascii") - return c - - escaped_string = "".join(map(escape_char, string)) - possible_quotes = quote_types - if "\n" in escaped_string: - possible_quotes = [q for q in possible_quotes if q in _MULTI_QUOTES] - possible_quotes = [q for q in possible_quotes if q not in escaped_string] - if not possible_quotes: - # If there aren't any possible_quotes, fallback to using repr - # on the original string. Try to use a quote from quote_types, - # e.g., so that we use triple quotes for docstrings. - string = repr(string) - quote = next((q for q in quote_types if string[0] in q), string[0]) - return string[1:-1], [quote] - if escaped_string: - # Sort so that we prefer '''"''' over """\"""" - possible_quotes.sort(key=lambda q: q[0] == escaped_string[-1]) - # If we're using triple quotes and we'd need to escape a final - # quote, escape it - if possible_quotes[0][0] == escaped_string[-1]: - assert len(possible_quotes[0]) == 3 - escaped_string = escaped_string[:-1] + "\\" + escaped_string[-1] - return escaped_string, possible_quotes - - def _write_str_avoiding_backslashes(self, string, *, quote_types=_ALL_QUOTES): - """Write string literal value with a best effort attempt to avoid backslashes.""" - string, quote_types = self._str_literal_helper(string, quote_types=quote_types) - quote_type = quote_types[0] - self.write(f"{quote_type}{string}{quote_type}") - - def visit_JoinedStr(self, node): - self.write("f") - - fstring_parts = [] - for value in node.values: - with self.buffered() as buffer: - self._write_fstring_inner(value) - fstring_parts.append( - ("".join(buffer), isinstance(value, Constant)) - ) - - new_fstring_parts = [] - quote_types = list(_ALL_QUOTES) - fallback_to_repr = False - for value, is_constant in fstring_parts: - if is_constant: - value, new_quote_types = self._str_literal_helper( - value, - quote_types=quote_types, - escape_special_whitespace=True, - ) - if set(new_quote_types).isdisjoint(quote_types): - fallback_to_repr = True - break - quote_types = new_quote_types - else: - if "\n" in value: - quote_types = [q for q in quote_types if q in _MULTI_QUOTES] - assert quote_types - - new_quote_types = [q for q in quote_types if q not in value] - if new_quote_types: - quote_types = new_quote_types - new_fstring_parts.append(value) - - if fallback_to_repr: - # If we weren't able to find a quote type that works for all parts - # of the JoinedStr, fallback to using repr and triple single quotes. - quote_types = ["'''"] - new_fstring_parts.clear() - for value, is_constant in fstring_parts: - if is_constant: - value = repr('"' + value) # force repr to use single quotes - expected_prefix = "'\"" - assert value.startswith(expected_prefix), repr(value) - value = value[len(expected_prefix):-1] - new_fstring_parts.append(value) - - value = "".join(new_fstring_parts) - quote_type = quote_types[0] - self.write(f"{quote_type}{value}{quote_type}") - - def _write_fstring_inner(self, node, is_format_spec=False): - if isinstance(node, JoinedStr): - # for both the f-string itself, and format_spec - for value in node.values: - self._write_fstring_inner(value, is_format_spec=is_format_spec) - elif isinstance(node, Constant) and isinstance(node.value, str): - value = node.value.replace("{", "{{").replace("}", "}}") - - if is_format_spec: - value = value.replace("\\", "\\\\") - value = value.replace("'", "\\'") - value = value.replace('"', '\\"') - value = value.replace("\n", "\\n") - self.write(value) - elif isinstance(node, FormattedValue): - self.visit_FormattedValue(node) - else: - raise ValueError(f"Unexpected node inside JoinedStr, {node!r}") - - def visit_FormattedValue(self, node): - def unparse_inner(inner): - unparser = type(self)() - unparser.set_precedence(_Precedence.TEST.next(), inner) - return unparser.visit(inner) - - with self.delimit("{", "}"): - expr = unparse_inner(node.value) - if expr.startswith("{"): - # Separate pair of opening brackets as "{ {" - self.write(" ") - self.write(expr) - if node.conversion != -1: - self.write(f"!{chr(node.conversion)}") - if node.format_spec: - self.write(":") - self._write_fstring_inner(node.format_spec, is_format_spec=True) - - def visit_Name(self, node): - self.write(node.id) - - def _write_docstring(self, node): - self.fill() - if node.kind == "u": - self.write("u") - self._write_str_avoiding_backslashes(node.value, quote_types=_MULTI_QUOTES) - - def _write_constant(self, value): - if isinstance(value, (float, complex)): - # Substitute overflowing decimal literal for AST infinities, - # and inf - inf for NaNs. - self.write( - repr(value) - .replace("inf", _INFSTR) - .replace("nan", f"({_INFSTR}-{_INFSTR})") - ) - else: - self.write(repr(value)) - - def visit_Constant(self, node): - value = node.value - if isinstance(value, tuple): - with self.delimit("(", ")"): - self.items_view(self._write_constant, value) - elif value is ...: - self.write("...") - else: - if node.kind == "u": - self.write("u") - self._write_constant(node.value) - - def visit_List(self, node): - with self.delimit("[", "]"): - self.interleave(lambda: self.write(", "), self.traverse, node.elts) - - def visit_ListComp(self, node): - with self.delimit("[", "]"): - self.traverse(node.elt) - for gen in node.generators: - self.traverse(gen) - - def visit_GeneratorExp(self, node): - with self.delimit("(", ")"): - self.traverse(node.elt) - for gen in node.generators: - self.traverse(gen) - - def visit_SetComp(self, node): - with self.delimit("{", "}"): - self.traverse(node.elt) - for gen in node.generators: - self.traverse(gen) - - def visit_DictComp(self, node): - with self.delimit("{", "}"): - self.traverse(node.key) - self.write(": ") - self.traverse(node.value) - for gen in node.generators: - self.traverse(gen) - - def visit_comprehension(self, node): - if node.is_async: - self.write(" async for ") - else: - self.write(" for ") - self.set_precedence(_Precedence.TUPLE, node.target) - self.traverse(node.target) - self.write(" in ") - self.set_precedence(_Precedence.TEST.next(), node.iter, *node.ifs) - self.traverse(node.iter) - for if_clause in node.ifs: - self.write(" if ") - self.traverse(if_clause) - - def visit_IfExp(self, node): - with self.require_parens(_Precedence.TEST, node): - self.set_precedence(_Precedence.TEST.next(), node.body, node.test) - self.traverse(node.body) - self.write(" if ") - self.traverse(node.test) - self.write(" else ") - self.set_precedence(_Precedence.TEST, node.orelse) - self.traverse(node.orelse) - - def visit_Set(self, node): - if node.elts: - with self.delimit("{", "}"): - self.interleave(lambda: self.write(", "), self.traverse, node.elts) - else: - # `{}` would be interpreted as a dictionary literal, and - # `set` might be shadowed. Thus: - self.write('{*()}') - - def visit_Dict(self, node): - def write_key_value_pair(k, v): - self.traverse(k) - self.write(": ") - self.traverse(v) - - def write_item(item): - k, v = item - if k is None: - # for dictionary unpacking operator in dicts {**{'y': 2}} - # see PEP 448 for details - self.write("**") - self.set_precedence(_Precedence.EXPR, v) - self.traverse(v) - else: - write_key_value_pair(k, v) - - with self.delimit("{", "}"): - self.interleave( - lambda: self.write(", "), write_item, zip(node.keys, node.values) - ) - - def visit_Tuple(self, node): - with self.delimit_if( - "(", - ")", - len(node.elts) == 0 or self.get_precedence(node) > _Precedence.TUPLE - ): - self.items_view(self.traverse, node.elts) - - unop = {"Invert": "~", "Not": "not", "UAdd": "+", "USub": "-"} - unop_precedence = { - "not": _Precedence.NOT, - "~": _Precedence.FACTOR, - "+": _Precedence.FACTOR, - "-": _Precedence.FACTOR, - } - - def visit_UnaryOp(self, node): - operator = self.unop[node.op.__class__.__name__] - operator_precedence = self.unop_precedence[operator] - with self.require_parens(operator_precedence, node): - self.write(operator) - # factor prefixes (+, -, ~) shouldn't be separated - # from the value they belong, (e.g: +1 instead of + 1) - if operator_precedence is not _Precedence.FACTOR: - self.write(" ") - self.set_precedence(operator_precedence, node.operand) - self.traverse(node.operand) - - binop = { - "Add": "+", - "Sub": "-", - "Mult": "*", - "MatMult": "@", - "Div": "/", - "Mod": "%", - "LShift": "<<", - "RShift": ">>", - "BitOr": "|", - "BitXor": "^", - "BitAnd": "&", - "FloorDiv": "//", - "Pow": "**", - } - - binop_precedence = { - "+": _Precedence.ARITH, - "-": _Precedence.ARITH, - "*": _Precedence.TERM, - "@": _Precedence.TERM, - "/": _Precedence.TERM, - "%": _Precedence.TERM, - "<<": _Precedence.SHIFT, - ">>": _Precedence.SHIFT, - "|": _Precedence.BOR, - "^": _Precedence.BXOR, - "&": _Precedence.BAND, - "//": _Precedence.TERM, - "**": _Precedence.POWER, - } - - binop_rassoc = frozenset(("**",)) - def visit_BinOp(self, node): - operator = self.binop[node.op.__class__.__name__] - operator_precedence = self.binop_precedence[operator] - with self.require_parens(operator_precedence, node): - if operator in self.binop_rassoc: - left_precedence = operator_precedence.next() - right_precedence = operator_precedence - else: - left_precedence = operator_precedence - right_precedence = operator_precedence.next() - - self.set_precedence(left_precedence, node.left) - self.traverse(node.left) - self.write(f" {operator} ") - self.set_precedence(right_precedence, node.right) - self.traverse(node.right) - - cmpops = { - "Eq": "==", - "NotEq": "!=", - "Lt": "<", - "LtE": "<=", - "Gt": ">", - "GtE": ">=", - "Is": "is", - "IsNot": "is not", - "In": "in", - "NotIn": "not in", - } - - def visit_Compare(self, node): - with self.require_parens(_Precedence.CMP, node): - self.set_precedence(_Precedence.CMP.next(), node.left, *node.comparators) - self.traverse(node.left) - for o, e in zip(node.ops, node.comparators): - self.write(" " + self.cmpops[o.__class__.__name__] + " ") - self.traverse(e) - - boolops = {"And": "and", "Or": "or"} - boolop_precedence = {"and": _Precedence.AND, "or": _Precedence.OR} - - def visit_BoolOp(self, node): - operator = self.boolops[node.op.__class__.__name__] - operator_precedence = self.boolop_precedence[operator] - - def increasing_level_traverse(node): - nonlocal operator_precedence - operator_precedence = operator_precedence.next() - self.set_precedence(operator_precedence, node) - self.traverse(node) - - with self.require_parens(operator_precedence, node): - s = f" {operator} " - self.interleave(lambda: self.write(s), increasing_level_traverse, node.values) - - def visit_Attribute(self, node): - self.set_precedence(_Precedence.ATOM, node.value) - self.traverse(node.value) - # Special case: 3.__abs__() is a syntax error, so if node.value - # is an integer literal then we need to either parenthesize - # it or add an extra space to get 3 .__abs__(). - if isinstance(node.value, Constant) and isinstance(node.value.value, int): - self.write(" ") - self.write(".") - self.write(node.attr) - - def visit_Call(self, node): - self.set_precedence(_Precedence.ATOM, node.func) - self.traverse(node.func) - with self.delimit("(", ")"): - comma = False - for e in node.args: - if comma: - self.write(", ") - else: - comma = True - self.traverse(e) - for e in node.keywords: - if comma: - self.write(", ") - else: - comma = True - self.traverse(e) - - def visit_Subscript(self, node): - def is_non_empty_tuple(slice_value): - return ( - isinstance(slice_value, Tuple) - and slice_value.elts - ) - - self.set_precedence(_Precedence.ATOM, node.value) - self.traverse(node.value) - with self.delimit("[", "]"): - if is_non_empty_tuple(node.slice): - # parentheses can be omitted if the tuple isn't empty - self.items_view(self.traverse, node.slice.elts) - else: - self.traverse(node.slice) - - def visit_Starred(self, node): - self.write("*") - self.set_precedence(_Precedence.EXPR, node.value) - self.traverse(node.value) - - def visit_Ellipsis(self, node): - self.write("...") - - def visit_Slice(self, node): - if node.lower: - self.traverse(node.lower) - self.write(":") - if node.upper: - self.traverse(node.upper) - if node.step: - self.write(":") - self.traverse(node.step) - - def visit_Match(self, node): - self.fill("match ") - self.traverse(node.subject) - with self.block(): - for case in node.cases: - self.traverse(case) - - def visit_arg(self, node): - self.write(node.arg) - if node.annotation: - self.write(": ") - self.traverse(node.annotation) - - def visit_arguments(self, node): - first = True - # normal arguments - all_args = node.posonlyargs + node.args - defaults = [None] * (len(all_args) - len(node.defaults)) + node.defaults - for index, elements in enumerate(zip(all_args, defaults), 1): - a, d = elements - if first: - first = False - else: - self.write(", ") - self.traverse(a) - if d: - self.write("=") - self.traverse(d) - if index == len(node.posonlyargs): - self.write(", /") - - # varargs, or bare '*' if no varargs but keyword-only arguments present - if node.vararg or node.kwonlyargs: - if first: - first = False - else: - self.write(", ") - self.write("*") - if node.vararg: - self.write(node.vararg.arg) - if node.vararg.annotation: - self.write(": ") - self.traverse(node.vararg.annotation) - - # keyword-only arguments - if node.kwonlyargs: - for a, d in zip(node.kwonlyargs, node.kw_defaults): - self.write(", ") - self.traverse(a) - if d: - self.write("=") - self.traverse(d) - - # kwargs - if node.kwarg: - if first: - first = False - else: - self.write(", ") - self.write("**" + node.kwarg.arg) - if node.kwarg.annotation: - self.write(": ") - self.traverse(node.kwarg.annotation) - - def visit_keyword(self, node): - if node.arg is None: - self.write("**") - else: - self.write(node.arg) - self.write("=") - self.traverse(node.value) - - def visit_Lambda(self, node): - with self.require_parens(_Precedence.TEST, node): - self.write("lambda") - with self.buffered() as buffer: - self.traverse(node.args) - if buffer: - self.write(" ", *buffer) - self.write(": ") - self.set_precedence(_Precedence.TEST, node.body) - self.traverse(node.body) - - def visit_alias(self, node): - self.write(node.name) - if node.asname: - self.write(" as " + node.asname) - - def visit_withitem(self, node): - self.traverse(node.context_expr) - if node.optional_vars: - self.write(" as ") - self.traverse(node.optional_vars) - - def visit_match_case(self, node): - self.fill("case ") - self.traverse(node.pattern) - if node.guard: - self.write(" if ") - self.traverse(node.guard) - with self.block(): - self.traverse(node.body) - - def visit_MatchValue(self, node): - self.traverse(node.value) - - def visit_MatchSingleton(self, node): - self._write_constant(node.value) - - def visit_MatchSequence(self, node): - with self.delimit("[", "]"): - self.interleave( - lambda: self.write(", "), self.traverse, node.patterns - ) - - def visit_MatchStar(self, node): - name = node.name - if name is None: - name = "_" - self.write(f"*{name}") - - def visit_MatchMapping(self, node): - def write_key_pattern_pair(pair): - k, p = pair - self.traverse(k) - self.write(": ") - self.traverse(p) - - with self.delimit("{", "}"): - keys = node.keys - self.interleave( - lambda: self.write(", "), - write_key_pattern_pair, - zip(keys, node.patterns, strict=True), - ) - rest = node.rest - if rest is not None: - if keys: - self.write(", ") - self.write(f"**{rest}") - - def visit_MatchClass(self, node): - self.set_precedence(_Precedence.ATOM, node.cls) - self.traverse(node.cls) - with self.delimit("(", ")"): - patterns = node.patterns - self.interleave( - lambda: self.write(", "), self.traverse, patterns - ) - attrs = node.kwd_attrs - if attrs: - def write_attr_pattern(pair): - attr, pattern = pair - self.write(f"{attr}=") - self.traverse(pattern) - - if patterns: - self.write(", ") - self.interleave( - lambda: self.write(", "), - write_attr_pattern, - zip(attrs, node.kwd_patterns, strict=True), - ) - - def visit_MatchAs(self, node): - name = node.name - pattern = node.pattern - if name is None: - self.write("_") - elif pattern is None: - self.write(node.name) - else: - with self.require_parens(_Precedence.TEST, node): - self.set_precedence(_Precedence.BOR, node.pattern) - self.traverse(node.pattern) - self.write(f" as {node.name}") - - def visit_MatchOr(self, node): - with self.require_parens(_Precedence.BOR, node): - self.set_precedence(_Precedence.BOR.next(), *node.patterns) - self.interleave(lambda: self.write(" | "), self.traverse, node.patterns) - def unparse(ast_obj): - unparser = _Unparser() + global _Unparser + try: + unparser = _Unparser() + except NameError: + from _ast_unparse import Unparser as _Unparser + unparser = _Unparser() return unparser.visit(ast_obj) -_deprecated_globals = { - name: globals().pop(name) - for name in ('Num', 'Str', 'Bytes', 'NameConstant', 'Ellipsis') -} - -def __getattr__(name): - if name in _deprecated_globals: - globals()[name] = value = _deprecated_globals[name] - import warnings - warnings._deprecated( - f"ast.{name}", message=_DEPRECATED_CLASS_MESSAGE, remove=(3, 14) - ) - return value - raise AttributeError(f"module 'ast' has no attribute '{name}'") - - -def main(): +def main(args=None): import argparse + import sys - parser = argparse.ArgumentParser(prog='python -m ast') + parser = argparse.ArgumentParser(color=True) parser.add_argument('infile', nargs='?', default='-', help='the file to parse; defaults to stdin') parser.add_argument('-m', '--mode', default='exec', @@ -1849,7 +641,16 @@ def main(): 'column offsets') parser.add_argument('-i', '--indent', type=int, default=3, help='indentation of nodes (number of spaces)') - args = parser.parse_args() + parser.add_argument('--feature-version', + type=str, default=None, metavar='VERSION', + help='Python version in the format 3.x ' + '(for example, 3.10)') + parser.add_argument('-O', '--optimize', + type=int, default=-1, metavar='LEVEL', + help='optimization level for parser (default -1)') + parser.add_argument('--show-empty', default=False, action='store_true', + help='show empty lists and fields in dump output') + args = parser.parse_args(args) if args.infile == '-': name = '' @@ -1858,8 +659,22 @@ def main(): name = args.infile with open(args.infile, 'rb') as infile: source = infile.read() - tree = parse(source, name, args.mode, type_comments=args.no_type_comments) - print(dump(tree, include_attributes=args.include_attributes, indent=args.indent)) + + # Process feature_version + feature_version = None + if args.feature_version: + try: + major, minor = map(int, args.feature_version.split('.', 1)) + except ValueError: + parser.error('Invalid format for --feature-version; ' + 'expected format 3.x (for example, 3.10)') + + feature_version = (major, minor) + + tree = parse(source, name, args.mode, type_comments=args.no_type_comments, + feature_version=feature_version, optimize=args.optimize) + print(dump(tree, include_attributes=args.include_attributes, + indent=args.indent, show_empty=args.show_empty)) if __name__ == '__main__': main() diff --git a/Lib/asyncio/__init__.py b/Lib/asyncio/__init__.py index 03165a425eb..32a5dbae03a 100644 --- a/Lib/asyncio/__init__.py +++ b/Lib/asyncio/__init__.py @@ -10,6 +10,7 @@ from .events import * from .exceptions import * from .futures import * +from .graph import * from .locks import * from .protocols import * from .runners import * @@ -27,6 +28,7 @@ events.__all__ + exceptions.__all__ + futures.__all__ + + graph.__all__ + locks.__all__ + protocols.__all__ + runners.__all__ + @@ -45,3 +47,28 @@ else: from .unix_events import * # pragma: no cover __all__ += unix_events.__all__ + +def __getattr__(name: str): + import warnings + + match name: + case "AbstractEventLoopPolicy": + warnings._deprecated(f"asyncio.{name}", remove=(3, 16)) + return events._AbstractEventLoopPolicy + case "DefaultEventLoopPolicy": + warnings._deprecated(f"asyncio.{name}", remove=(3, 16)) + if sys.platform == 'win32': + return windows_events._DefaultEventLoopPolicy + return unix_events._DefaultEventLoopPolicy + case "WindowsSelectorEventLoopPolicy": + if sys.platform == 'win32': + warnings._deprecated(f"asyncio.{name}", remove=(3, 16)) + return windows_events._WindowsSelectorEventLoopPolicy + # Else fall through to the AttributeError below. + case "WindowsProactorEventLoopPolicy": + if sys.platform == 'win32': + warnings._deprecated(f"asyncio.{name}", remove=(3, 16)) + return windows_events._WindowsProactorEventLoopPolicy + # Else fall through to the AttributeError below. + + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Lib/asyncio/__main__.py b/Lib/asyncio/__main__.py index 8e9af83cc8a..e07dd52a2a5 100644 --- a/Lib/asyncio/__main__.py +++ b/Lib/asyncio/__main__.py @@ -1,5 +1,7 @@ +import argparse import ast import asyncio +import asyncio.tools import concurrent.futures import contextvars import inspect @@ -10,7 +12,7 @@ import types import warnings -from _colorize import can_colorize, ANSIColors # type: ignore[import-not-found] +from _colorize import get_theme from _pyrepl.console import InteractiveColoredConsole from . import futures @@ -102,8 +104,9 @@ def run(self): exec(startup_code, console.locals) ps1 = getattr(sys, "ps1", ">>> ") - if can_colorize() and CAN_USE_PYREPL: - ps1 = f"{ANSIColors.BOLD_MAGENTA}{ps1}{ANSIColors.RESET}" + if CAN_USE_PYREPL: + theme = get_theme().syntax + ps1 = f"{theme.prompt}{ps1}{theme.reset}" console.write(f"{ps1}import asyncio\n") if CAN_USE_PYREPL: @@ -141,6 +144,37 @@ def interrupt(self) -> None: if __name__ == '__main__': + parser = argparse.ArgumentParser( + prog="python3 -m asyncio", + description="Interactive asyncio shell and CLI tools", + color=True, + ) + subparsers = parser.add_subparsers(help="sub-commands", dest="command") + ps = subparsers.add_parser( + "ps", help="Display a table of all pending tasks in a process" + ) + ps.add_argument("pid", type=int, help="Process ID to inspect") + pstree = subparsers.add_parser( + "pstree", help="Display a tree of all pending tasks in a process" + ) + pstree.add_argument("pid", type=int, help="Process ID to inspect") + args = parser.parse_args() + match args.command: + case "ps": + asyncio.tools.display_awaited_by_tasks_table(args.pid) + sys.exit(0) + case "pstree": + asyncio.tools.display_awaited_by_tasks_tree(args.pid) + sys.exit(0) + case None: + pass # continue to the interactive shell + case _: + # shouldn't happen as an invalid command-line wouldn't parse + # but let's keep it for the next person adding a command + print(f"error: unhandled command {args.command}", file=sys.stderr) + parser.print_usage(file=sys.stderr) + sys.exit(1) + sys.audit("cpython.run_stdin") if os.getenv('PYTHON_BASIC_REPL'): diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index 356fc3d7913..8cbb71f7085 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -458,24 +458,18 @@ def create_future(self): """Create a Future object attached to the loop.""" return futures.Future(loop=self) - def create_task(self, coro, *, name=None, context=None, **kwargs): - """Schedule a coroutine object. + def create_task(self, coro, **kwargs): + """Schedule or begin executing a coroutine object. Return a task object. """ self._check_closed() if self._task_factory is not None: - if context is not None: - kwargs["context"] = context - - task = self._task_factory(self, coro, **kwargs) - task.set_name(name) - - else: - task = tasks.Task(coro, loop=self, name=name, context=context, **kwargs) - if task._source_traceback: - del task._source_traceback[-1] + return self._task_factory(self, coro, **kwargs) + task = tasks.Task(coro, loop=self, **kwargs) + if task._source_traceback: + del task._source_traceback[-1] try: return task finally: @@ -841,7 +835,7 @@ def call_soon(self, callback, *args, context=None): def _check_callback(self, callback, method): if (coroutines.iscoroutine(callback) or - coroutines.iscoroutinefunction(callback)): + coroutines._iscoroutinefunction(callback)): raise TypeError( f"coroutines cannot be used with {method}()") if not callable(callback): @@ -878,7 +872,10 @@ def call_soon_threadsafe(self, callback, *args, context=None): self._check_closed() if self._debug: self._check_callback(callback, 'call_soon_threadsafe') - handle = self._call_soon(callback, args, context) + handle = events._ThreadSafeHandle(callback, args, self, context) + self._ready.append(handle) + if handle._source_traceback: + del handle._source_traceback[-1] if handle._source_traceback: del handle._source_traceback[-1] self._write_to_self() @@ -1677,8 +1674,7 @@ async def connect_accepted_socket( raise ValueError( 'ssl_shutdown_timeout is only meaningful with ssl') - if sock is not None: - _check_ssl_socket(sock) + _check_ssl_socket(sock) transport, protocol = await self._create_connection_transport( sock, protocol_factory, ssl, '', server_side=True, diff --git a/Lib/asyncio/coroutines.py b/Lib/asyncio/coroutines.py index ab4f30eb51b..a51319cb72a 100644 --- a/Lib/asyncio/coroutines.py +++ b/Lib/asyncio/coroutines.py @@ -18,7 +18,16 @@ def _is_debug_mode(): def iscoroutinefunction(func): + import warnings """Return True if func is a decorated coroutine function.""" + warnings._deprecated("asyncio.iscoroutinefunction", + f"{warnings._DEPRECATED_MSG}; " + "use inspect.iscoroutinefunction() instead", + remove=(3,16)) + return _iscoroutinefunction(func) + + +def _iscoroutinefunction(func): return (inspect.iscoroutinefunction(func) or getattr(func, '_is_coroutine', None) is _is_coroutine) diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index 0d1591e4604..a7fb55982ab 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -5,14 +5,18 @@ # SPDX-FileCopyrightText: Copyright (c) 2015-2021 MagicStack Inc. http://magic.io __all__ = ( - 'AbstractEventLoopPolicy', - 'AbstractEventLoop', 'AbstractServer', - 'Handle', 'TimerHandle', - 'get_event_loop_policy', 'set_event_loop_policy', - 'get_event_loop', 'set_event_loop', 'new_event_loop', - 'get_child_watcher', 'set_child_watcher', - '_set_running_loop', 'get_running_loop', - '_get_running_loop', + "AbstractEventLoop", + "AbstractServer", + "Handle", + "TimerHandle", + "get_event_loop_policy", + "set_event_loop_policy", + "get_event_loop", + "set_event_loop", + "new_event_loop", + "_set_running_loop", + "get_running_loop", + "_get_running_loop", ) import contextvars @@ -22,6 +26,7 @@ import subprocess import sys import threading +import warnings from . import format_helpers @@ -104,6 +109,34 @@ def _run(self): self._loop.call_exception_handler(context) self = None # Needed to break cycles when an exception occurs. +# _ThreadSafeHandle is used for callbacks scheduled with call_soon_threadsafe +# and is thread safe unlike Handle which is not thread safe. +class _ThreadSafeHandle(Handle): + + __slots__ = ('_lock',) + + def __init__(self, callback, args, loop, context=None): + super().__init__(callback, args, loop, context) + self._lock = threading.RLock() + + def cancel(self): + with self._lock: + return super().cancel() + + def cancelled(self): + with self._lock: + return super().cancelled() + + def _run(self): + # The event loop checks for cancellation without holding the lock + # It is possible that the handle is cancelled after the check + # but before the callback is called so check it again after acquiring + # the lock and return without calling the callback if it is cancelled. + with self._lock: + if self._cancelled: + return + return super()._run() + class TimerHandle(Handle): """Object returned by timed callback registration methods.""" @@ -629,7 +662,7 @@ def set_debug(self, enabled): raise NotImplementedError -class AbstractEventLoopPolicy: +class _AbstractEventLoopPolicy: """Abstract policy for accessing the event loop.""" def get_event_loop(self): @@ -652,18 +685,7 @@ def new_event_loop(self): the current context, set_event_loop must be called explicitly.""" raise NotImplementedError - # Child processes handling (Unix only). - - def get_child_watcher(self): - "Get the watcher for child processes." - raise NotImplementedError - - def set_child_watcher(self, watcher): - """Set the watcher for child processes.""" - raise NotImplementedError - - -class BaseDefaultEventLoopPolicy(AbstractEventLoopPolicy): +class _BaseDefaultEventLoopPolicy(_AbstractEventLoopPolicy): """Default policy implementation for accessing the event loop. In this policy, each thread has its own event loop. However, we @@ -680,7 +702,6 @@ class BaseDefaultEventLoopPolicy(AbstractEventLoopPolicy): class _Local(threading.local): _loop = None - _set_called = False def __init__(self): self._local = self._Local() @@ -690,28 +711,6 @@ def get_event_loop(self): Returns an instance of EventLoop or raises an exception. """ - if (self._local._loop is None and - not self._local._set_called and - threading.current_thread() is threading.main_thread()): - stacklevel = 2 - try: - f = sys._getframe(1) - except AttributeError: - pass - else: - # Move up the call stack so that the warning is attached - # to the line outside asyncio itself. - while f: - module = f.f_globals.get('__name__') - if not (module == 'asyncio' or module.startswith('asyncio.')): - break - f = f.f_back - stacklevel += 1 - import warnings - warnings.warn('There is no current event loop', - DeprecationWarning, stacklevel=stacklevel) - self.set_event_loop(self.new_event_loop()) - if self._local._loop is None: raise RuntimeError('There is no current event loop in thread %r.' % threading.current_thread().name) @@ -720,7 +719,6 @@ def get_event_loop(self): def set_event_loop(self, loop): """Set the event loop.""" - self._local._set_called = True if loop is not None and not isinstance(loop, AbstractEventLoop): raise TypeError(f"loop must be an instance of AbstractEventLoop or None, not '{type(loop).__name__}'") self._local._loop = loop @@ -790,34 +788,36 @@ def _init_event_loop_policy(): global _event_loop_policy with _lock: if _event_loop_policy is None: # pragma: no branch - from . import DefaultEventLoopPolicy - _event_loop_policy = DefaultEventLoopPolicy() + if sys.platform == 'win32': + from .windows_events import _DefaultEventLoopPolicy + else: + from .unix_events import _DefaultEventLoopPolicy + _event_loop_policy = _DefaultEventLoopPolicy() -def get_event_loop_policy(): +def _get_event_loop_policy(): """Get the current event loop policy.""" if _event_loop_policy is None: _init_event_loop_policy() return _event_loop_policy +def get_event_loop_policy(): + warnings._deprecated('asyncio.get_event_loop_policy', remove=(3, 16)) + return _get_event_loop_policy() def _set_event_loop_policy(policy): """Set the current event loop policy. If policy is None, the default policy is restored.""" global _event_loop_policy - if policy is not None and not isinstance(policy, AbstractEventLoopPolicy): + if policy is not None and not isinstance(policy, _AbstractEventLoopPolicy): raise TypeError(f"policy must be an instance of AbstractEventLoopPolicy or None, not '{type(policy).__name__}'") _event_loop_policy = policy - def set_event_loop_policy(policy): - """Set the current event loop policy. - - If policy is None, the default policy is restored.""" + warnings._deprecated('asyncio.set_event_loop_policy', remove=(3,16)) _set_event_loop_policy(policy) - def get_event_loop(): """Return an asyncio event loop. @@ -831,28 +831,17 @@ def get_event_loop(): current_loop = _get_running_loop() if current_loop is not None: return current_loop - return get_event_loop_policy().get_event_loop() + return _get_event_loop_policy().get_event_loop() def set_event_loop(loop): """Equivalent to calling get_event_loop_policy().set_event_loop(loop).""" - get_event_loop_policy().set_event_loop(loop) + _get_event_loop_policy().set_event_loop(loop) def new_event_loop(): """Equivalent to calling get_event_loop_policy().new_event_loop().""" - return get_event_loop_policy().new_event_loop() - - -def get_child_watcher(): - """Equivalent to calling get_event_loop_policy().get_child_watcher().""" - return get_event_loop_policy().get_child_watcher() - - -def set_child_watcher(watcher): - """Equivalent to calling - get_event_loop_policy().set_child_watcher(watcher).""" - return get_event_loop_policy().set_child_watcher(watcher) + return _get_event_loop_policy().new_event_loop() # Alias pure-Python implementations for testing purposes. @@ -882,7 +871,7 @@ def set_child_watcher(watcher): def on_fork(): # Reset the loop and wakeupfd in the forked child process. if _event_loop_policy is not None: - _event_loop_policy._local = BaseDefaultEventLoopPolicy._Local() + _event_loop_policy._local = _BaseDefaultEventLoopPolicy._Local() _set_running_loop(None) signal.set_wakeup_fd(-1) diff --git a/Lib/asyncio/futures.py b/Lib/asyncio/futures.py index 51932639097..d1df6707302 100644 --- a/Lib/asyncio/futures.py +++ b/Lib/asyncio/futures.py @@ -2,6 +2,7 @@ __all__ = ( 'Future', 'wrap_future', 'isfuture', + 'future_add_to_awaited_by', 'future_discard_from_awaited_by', ) import concurrent.futures @@ -43,7 +44,6 @@ class Future: - This class is not compatible with the wait() and as_completed() methods in the concurrent.futures package. - (In Python 3.4 or later we may be able to unify the implementations.) """ # Class variables serving as defaults for instance variables. @@ -61,12 +61,15 @@ class Future: # the Future protocol (i.e. is intended to be duck-type compatible). # The value must also be not-None, to enable a subclass to declare # that it is not compatible by setting this to None. - # - It is set by __iter__() below so that Task._step() can tell + # - It is set by __iter__() below so that Task.__step() can tell # the difference between - # `await Future()` or`yield from Future()` (correct) vs. + # `await Future()` or `yield from Future()` (correct) vs. # `yield Future()` (incorrect). _asyncio_future_blocking = False + # Used by the capture_call_stack() API. + __asyncio_awaited_by = None + __log_traceback = False def __init__(self, *, loop=None): @@ -116,6 +119,12 @@ def _log_traceback(self, val): raise ValueError('_log_traceback can only be set to False') self.__log_traceback = False + @property + def _asyncio_awaited_by(self): + if self.__asyncio_awaited_by is None: + return None + return frozenset(self.__asyncio_awaited_by) + def get_loop(self): """Return the event loop the Future is bound to.""" loop = self._loop @@ -416,6 +425,49 @@ def wrap_future(future, *, loop=None): return new_future +def future_add_to_awaited_by(fut, waiter, /): + """Record that `fut` is awaited on by `waiter`.""" + # For the sake of keeping the implementation minimal and assuming + # that most of asyncio users use the built-in Futures and Tasks + # (or their subclasses), we only support native Future objects + # and their subclasses. + # + # Longer version: tracking requires storing the caller-callee + # dependency somewhere. One obvious choice is to store that + # information right in the future itself in a dedicated attribute. + # This means that we'd have to require all duck-type compatible + # futures to implement a specific attribute used by asyncio for + # the book keeping. Another solution would be to store that in + # a global dictionary. The downside here is that that would create + # strong references and any scenario where the "add" call isn't + # followed by a "discard" call would lead to a memory leak. + # Using WeakDict would resolve that issue, but would complicate + # the C code (_asynciomodule.c). The bottom line here is that + # it's not clear that all this work would be worth the effort. + # + # Note that there's an accelerated version of this function + # shadowing this implementation later in this file. + if isinstance(fut, _PyFuture) and isinstance(waiter, _PyFuture): + if fut._Future__asyncio_awaited_by is None: + fut._Future__asyncio_awaited_by = set() + fut._Future__asyncio_awaited_by.add(waiter) + + +def future_discard_from_awaited_by(fut, waiter, /): + """Record that `fut` is no longer awaited on by `waiter`.""" + # See the comment in "future_add_to_awaited_by()" body for + # details on implementation. + # + # Note that there's an accelerated version of this function + # shadowing this implementation later in this file. + if isinstance(fut, _PyFuture) and isinstance(waiter, _PyFuture): + if fut._Future__asyncio_awaited_by is not None: + fut._Future__asyncio_awaited_by.discard(waiter) + + +_py_future_add_to_awaited_by = future_add_to_awaited_by +_py_future_discard_from_awaited_by = future_discard_from_awaited_by + try: import _asyncio except ImportError: @@ -423,3 +475,7 @@ def wrap_future(future, *, loop=None): else: # _CFuture is needed for tests. Future = _CFuture = _asyncio.Future + future_add_to_awaited_by = _asyncio.future_add_to_awaited_by + future_discard_from_awaited_by = _asyncio.future_discard_from_awaited_by + _c_future_add_to_awaited_by = future_add_to_awaited_by + _c_future_discard_from_awaited_by = future_discard_from_awaited_by diff --git a/Lib/asyncio/graph.py b/Lib/asyncio/graph.py new file mode 100644 index 00000000000..b5bfeb1630a --- /dev/null +++ b/Lib/asyncio/graph.py @@ -0,0 +1,276 @@ +"""Introspection utils for tasks call graphs.""" + +import dataclasses +import io +import sys +import types + +from . import events +from . import futures +from . import tasks + +__all__ = ( + 'capture_call_graph', + 'format_call_graph', + 'print_call_graph', + 'FrameCallGraphEntry', + 'FutureCallGraph', +) + +# Sadly, we can't re-use the traceback module's datastructures as those +# are tailored for error reporting, whereas we need to represent an +# async call graph. +# +# Going with pretty verbose names as we'd like to export them to the +# top level asyncio namespace, and want to avoid future name clashes. + + +@dataclasses.dataclass(frozen=True, slots=True) +class FrameCallGraphEntry: + frame: types.FrameType + + +@dataclasses.dataclass(frozen=True, slots=True) +class FutureCallGraph: + future: futures.Future + call_stack: tuple["FrameCallGraphEntry", ...] + awaited_by: tuple["FutureCallGraph", ...] + + +def _build_graph_for_future( + future: futures.Future, + *, + limit: int | None = None, +) -> FutureCallGraph: + if not isinstance(future, futures.Future): + raise TypeError( + f"{future!r} object does not appear to be compatible " + f"with asyncio.Future" + ) + + coro = None + if get_coro := getattr(future, 'get_coro', None): + coro = get_coro() if limit != 0 else None + + st: list[FrameCallGraphEntry] = [] + awaited_by: list[FutureCallGraph] = [] + + while coro is not None: + if hasattr(coro, 'cr_await'): + # A native coroutine or duck-type compatible iterator + st.append(FrameCallGraphEntry(coro.cr_frame)) + coro = coro.cr_await + elif hasattr(coro, 'ag_await'): + # A native async generator or duck-type compatible iterator + st.append(FrameCallGraphEntry(coro.cr_frame)) + coro = coro.ag_await + else: + break + + if future._asyncio_awaited_by: + for parent in future._asyncio_awaited_by: + awaited_by.append(_build_graph_for_future(parent, limit=limit)) + + if limit is not None: + if limit > 0: + st = st[:limit] + elif limit < 0: + st = st[limit:] + st.reverse() + return FutureCallGraph(future, tuple(st), tuple(awaited_by)) + + +def capture_call_graph( + future: futures.Future | None = None, + /, + *, + depth: int = 1, + limit: int | None = None, +) -> FutureCallGraph | None: + """Capture the async call graph for the current task or the provided Future. + + The graph is represented with three data structures: + + * FutureCallGraph(future, call_stack, awaited_by) + + Where 'future' is an instance of asyncio.Future or asyncio.Task. + + 'call_stack' is a tuple of FrameGraphEntry objects. + + 'awaited_by' is a tuple of FutureCallGraph objects. + + * FrameCallGraphEntry(frame) + + Where 'frame' is a frame object of a regular Python function + in the call stack. + + Receives an optional 'future' argument. If not passed, + the current task will be used. If there's no current task, the function + returns None. + + If "capture_call_graph()" is introspecting *the current task*, the + optional keyword-only 'depth' argument can be used to skip the specified + number of frames from top of the stack. + + If the optional keyword-only 'limit' argument is provided, each call stack + in the resulting graph is truncated to include at most ``abs(limit)`` + entries. If 'limit' is positive, the entries left are the closest to + the invocation point. If 'limit' is negative, the topmost entries are + left. If 'limit' is omitted or None, all entries are present. + If 'limit' is 0, the call stack is not captured at all, only + "awaited by" information is present. + """ + + loop = events._get_running_loop() + + if future is not None: + # Check if we're in a context of a running event loop; + # if yes - check if the passed future is the currently + # running task or not. + if loop is None or future is not tasks.current_task(loop=loop): + return _build_graph_for_future(future, limit=limit) + # else: future is the current task, move on. + else: + if loop is None: + raise RuntimeError( + 'capture_call_graph() is called outside of a running ' + 'event loop and no *future* to introspect was provided') + future = tasks.current_task(loop=loop) + + if future is None: + # This isn't a generic call stack introspection utility. If we + # can't determine the current task and none was provided, we + # just return. + return None + + if not isinstance(future, futures.Future): + raise TypeError( + f"{future!r} object does not appear to be compatible " + f"with asyncio.Future" + ) + + call_stack: list[FrameCallGraphEntry] = [] + + f = sys._getframe(depth) if limit != 0 else None + try: + while f is not None: + is_async = f.f_generator is not None + call_stack.append(FrameCallGraphEntry(f)) + + if is_async: + if f.f_back is not None and f.f_back.f_generator is None: + # We've reached the bottom of the coroutine stack, which + # must be the Task that runs it. + break + + f = f.f_back + finally: + del f + + awaited_by = [] + if future._asyncio_awaited_by: + for parent in future._asyncio_awaited_by: + awaited_by.append(_build_graph_for_future(parent, limit=limit)) + + if limit is not None: + limit *= -1 + if limit > 0: + call_stack = call_stack[:limit] + elif limit < 0: + call_stack = call_stack[limit:] + + return FutureCallGraph(future, tuple(call_stack), tuple(awaited_by)) + + +def format_call_graph( + future: futures.Future | None = None, + /, + *, + depth: int = 1, + limit: int | None = None, +) -> str: + """Return the async call graph as a string for `future`. + + If `future` is not provided, format the call graph for the current task. + """ + + def render_level(st: FutureCallGraph, buf: list[str], level: int) -> None: + def add_line(line: str) -> None: + buf.append(level * ' ' + line) + + if isinstance(st.future, tasks.Task): + add_line( + f'* Task(name={st.future.get_name()!r}, id={id(st.future):#x})' + ) + else: + add_line( + f'* Future(id={id(st.future):#x})' + ) + + if st.call_stack: + add_line( + f' + Call stack:' + ) + for ste in st.call_stack: + f = ste.frame + + if f.f_generator is None: + f = ste.frame + add_line( + f' | File {f.f_code.co_filename!r},' + f' line {f.f_lineno}, in' + f' {f.f_code.co_qualname}()' + ) + else: + c = f.f_generator + + try: + f = c.cr_frame + code = c.cr_code + tag = 'async' + except AttributeError: + try: + f = c.ag_frame + code = c.ag_code + tag = 'async generator' + except AttributeError: + f = c.gi_frame + code = c.gi_code + tag = 'generator' + + add_line( + f' | File {f.f_code.co_filename!r},' + f' line {f.f_lineno}, in' + f' {tag} {code.co_qualname}()' + ) + + if st.awaited_by: + add_line( + f' + Awaited by:' + ) + for fut in st.awaited_by: + render_level(fut, buf, level + 1) + + graph = capture_call_graph(future, depth=depth + 1, limit=limit) + if graph is None: + return "" + + buf: list[str] = [] + try: + render_level(graph, buf, 0) + finally: + # 'graph' has references to frames so we should + # make sure it's GC'ed as soon as we don't need it. + del graph + return '\n'.join(buf) + +def print_call_graph( + future: futures.Future | None = None, + /, + *, + file: io.Writer[str] | None = None, + depth: int = 1, + limit: int | None = None, +) -> None: + """Print the async call graph for the current task or the provided Future.""" + print(format_call_graph(future, depth=depth, limit=limit), file=file) diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py index 3df4c693a91..fa3a94764b5 100644 --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -341,9 +341,9 @@ def _notify(self, n): fut.set_result(False) def notify_all(self): - """Wake up all threads waiting on this condition. This method acts - like notify(), but wakes up all waiting threads instead of one. If the - calling thread has not acquired the lock when this method is called, + """Wake up all tasks waiting on this condition. This method acts + like notify(), but wakes up all waiting tasks instead of one. If the + calling task has not acquired the lock when this method is called, a RuntimeError is raised. """ self.notify(len(self._waiters)) diff --git a/Lib/asyncio/runners.py b/Lib/asyncio/runners.py index 102ae78021b..ba37e003a65 100644 --- a/Lib/asyncio/runners.py +++ b/Lib/asyncio/runners.py @@ -3,6 +3,7 @@ import contextvars import enum import functools +import inspect import threading import signal from . import coroutines @@ -84,10 +85,7 @@ def get_loop(self): return self._loop def run(self, coro, *, context=None): - """Run a coroutine inside the embedded event loop.""" - if not coroutines.iscoroutine(coro): - raise ValueError("a coroutine was expected, got {!r}".format(coro)) - + """Run code in the embedded event loop.""" if events._get_running_loop() is not None: # fail fast with short traceback raise RuntimeError( @@ -95,8 +93,19 @@ def run(self, coro, *, context=None): self._lazy_init() + if not coroutines.iscoroutine(coro): + if inspect.isawaitable(coro): + async def _wrap_awaitable(awaitable): + return await awaitable + + coro = _wrap_awaitable(coro) + else: + raise TypeError('An asyncio.Future, a coroutine or an ' + 'awaitable is required') + if context is None: context = self._context + task = self._loop.create_task(coro, context=context) if (threading.current_thread() is threading.main_thread() diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py index 8701467d413..ff7e16df3c6 100644 --- a/Lib/asyncio/selector_events.py +++ b/Lib/asyncio/selector_events.py @@ -173,16 +173,20 @@ def _accept_connection( # listening socket has triggered an EVENT_READ. There may be multiple # connections waiting for an .accept() so it is called in a loop. # See https://bugs.python.org/issue27906 for more details. - for _ in range(backlog): + for _ in range(backlog + 1): try: conn, addr = sock.accept() if self._debug: logger.debug("%r got a new connection from %r: %r", server, addr, conn) conn.setblocking(False) - except (BlockingIOError, InterruptedError, ConnectionAbortedError): - # Early exit because the socket accept buffer is empty. - return None + except ConnectionAbortedError: + # Discard connections that were aborted before accept(). + continue + except (BlockingIOError, InterruptedError): + # Early exit because of a signal or + # the socket accept buffer is empty. + return except OSError as exc: # There's nowhere to send the error, so just log it. if exc.errno in (errno.EMFILE, errno.ENFILE, diff --git a/Lib/asyncio/staggered.py b/Lib/asyncio/staggered.py index 0afed64fdf9..2ad65d8648e 100644 --- a/Lib/asyncio/staggered.py +++ b/Lib/asyncio/staggered.py @@ -8,6 +8,7 @@ from . import exceptions as exceptions_mod from . import locks from . import tasks +from . import futures async def staggered_race(coro_fns, delay, *, loop=None): @@ -63,6 +64,7 @@ async def staggered_race(coro_fns, delay, *, loop=None): """ # TODO: when we have aiter() and anext(), allow async iterables in coro_fns. loop = loop or events.get_running_loop() + parent_task = tasks.current_task(loop) enum_coro_fns = enumerate(coro_fns) winner_result = None winner_index = None @@ -73,6 +75,7 @@ async def staggered_race(coro_fns, delay, *, loop=None): def task_done(task): running_tasks.discard(task) + futures.future_discard_from_awaited_by(task, parent_task) if ( on_completed_fut is not None and not on_completed_fut.done() @@ -110,6 +113,7 @@ async def run_one_coro(ok_to_start, previous_failed) -> None: this_failed = locks.Event() next_ok_to_start = locks.Event() next_task = loop.create_task(run_one_coro(next_ok_to_start, this_failed)) + futures.future_add_to_awaited_by(next_task, parent_task) running_tasks.add(next_task) next_task.add_done_callback(task_done) # next_task has been appended to running_tasks so next_task is ok to @@ -148,6 +152,7 @@ async def run_one_coro(ok_to_start, previous_failed) -> None: try: ok_to_start = locks.Event() first_task = loop.create_task(run_one_coro(ok_to_start, None)) + futures.future_add_to_awaited_by(first_task, parent_task) running_tasks.add(first_task) first_task.add_done_callback(task_done) # first_task has been appended to running_tasks so first_task is ok to start. @@ -171,4 +176,4 @@ async def run_one_coro(ok_to_start, previous_failed) -> None: raise propagate_cancellation_error return winner_result, winner_index, exceptions finally: - del exceptions, propagate_cancellation_error, unhandled_exceptions + del exceptions, propagate_cancellation_error, unhandled_exceptions, parent_task diff --git a/Lib/asyncio/taskgroups.py b/Lib/asyncio/taskgroups.py index 8fda6c8d55e..00e8f6d5d1a 100644 --- a/Lib/asyncio/taskgroups.py +++ b/Lib/asyncio/taskgroups.py @@ -6,6 +6,7 @@ from . import events from . import exceptions +from . import futures from . import tasks @@ -178,7 +179,7 @@ async def _aexit(self, et, exc): exc = None - def create_task(self, coro, *, name=None, context=None): + def create_task(self, coro, **kwargs): """Create a new task in this group and return it. Similar to `asyncio.create_task`. @@ -192,10 +193,9 @@ def create_task(self, coro, *, name=None, context=None): if self._aborting: coro.close() raise RuntimeError(f"TaskGroup {self!r} is shutting down") - if context is None: - task = self._loop.create_task(coro, name=name) - else: - task = self._loop.create_task(coro, name=name, context=context) + task = self._loop.create_task(coro, **kwargs) + + futures.future_add_to_awaited_by(task, self._parent_task) # Always schedule the done callback even if the task is # already done (e.g. if the coro was able to complete eagerly), @@ -228,6 +228,8 @@ def _abort(self): def _on_task_done(self, task): self._tasks.discard(task) + futures.future_discard_from_awaited_by(task, self._parent_task) + if self._on_completed_fut is not None and not self._tasks: if not self._on_completed_fut.done(): self._on_completed_fut.set_result(True) diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index dadcb5b5f36..fbd5c39a7c5 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -48,23 +48,8 @@ def all_tasks(loop=None): # capturing the set of eager tasks first, so if an eager task "graduates" # to a regular task in another thread, we don't risk missing it. eager_tasks = list(_eager_tasks) - # Looping over the WeakSet isn't safe as it can be updated from another - # thread, therefore we cast it to list prior to filtering. The list cast - # itself requires iteration, so we repeat it several times ignoring - # RuntimeErrors (which are not very likely to occur). - # See issues 34970 and 36607 for details. - scheduled_tasks = None - i = 0 - while True: - try: - scheduled_tasks = list(_scheduled_tasks) - except RuntimeError: - i += 1 - if i >= 1000: - raise - else: - break - return {t for t in itertools.chain(scheduled_tasks, eager_tasks) + + return {t for t in itertools.chain(_scheduled_tasks, eager_tasks) if futures._get_loop(t) is loop and not t.done()} @@ -125,7 +110,7 @@ def __init__(self, coro, *, loop=None, name=None, context=None, self.__eager_start() else: self._loop.call_soon(self.__step, context=self._context) - _register_task(self) + _py_register_task(self) def __del__(self): if self._state == futures._PENDING and self._log_destroy_pending: @@ -260,39 +245,39 @@ def uncancel(self): return self._num_cancels_requested def __eager_start(self): - prev_task = _swap_current_task(self._loop, self) + prev_task = _py_swap_current_task(self._loop, self) try: - _register_eager_task(self) + _py_register_eager_task(self) try: self._context.run(self.__step_run_and_handle_result, None) finally: - _unregister_eager_task(self) + _py_unregister_eager_task(self) finally: try: - curtask = _swap_current_task(self._loop, prev_task) + curtask = _py_swap_current_task(self._loop, prev_task) assert curtask is self finally: if self.done(): self._coro = None self = None # Needed to break cycles when an exception occurs. else: - _register_task(self) + _py_register_task(self) def __step(self, exc=None): if self.done(): raise exceptions.InvalidStateError( - f'_step(): already done: {self!r}, {exc!r}') + f'__step(): already done: {self!r}, {exc!r}') if self._must_cancel: if not isinstance(exc, exceptions.CancelledError): exc = self._make_cancelled_error() self._must_cancel = False self._fut_waiter = None - _enter_task(self._loop, self) + _py_enter_task(self._loop, self) try: self.__step_run_and_handle_result(exc) finally: - _leave_task(self._loop, self) + _py_leave_task(self._loop, self) self = None # Needed to break cycles when an exception occurs. def __step_run_and_handle_result(self, exc): @@ -337,6 +322,7 @@ def __step_run_and_handle_result(self, exc): self._loop.call_soon( self.__step, new_exc, context=self._context) else: + futures.future_add_to_awaited_by(result, self) result._asyncio_future_blocking = False result.add_done_callback( self.__wakeup, context=self._context) @@ -371,6 +357,7 @@ def __step_run_and_handle_result(self, exc): self = None # Needed to break cycles when an exception occurs. def __wakeup(self, future): + futures.future_discard_from_awaited_by(future, self) try: future.result() except BaseException as exc: @@ -379,7 +366,7 @@ def __wakeup(self, future): else: # Don't pass the value of `future.result()` explicitly, # as `Future.__iter__` and `Future.__await__` don't need it. - # If we call `_step(value, None)` instead of `_step()`, + # If we call `__step(value, None)` instead of `__step()`, # Python eval loop would use `.send(value)` method call, # instead of `__next__()`, which is slower for futures # that return non-generator iterators from their `__iter__`. @@ -399,19 +386,13 @@ def __wakeup(self, future): Task = _CTask = _asyncio.Task -def create_task(coro, *, name=None, context=None): +def create_task(coro, **kwargs): """Schedule the execution of a coroutine object in a spawn task. Return a Task object. """ loop = events.get_running_loop() - if context is None: - # Use legacy API if context is not needed - task = loop.create_task(coro, name=name) - else: - task = loop.create_task(coro, name=name, context=context) - - return task + return loop.create_task(coro, **kwargs) # wait() and as_completed() similar to those in PEP 3148. @@ -517,6 +498,7 @@ async def _wait(fs, timeout, return_when, loop): if timeout is not None: timeout_handle = loop.call_later(timeout, _release_waiter, waiter) counter = len(fs) + cur_task = current_task() def _on_completion(f): nonlocal counter @@ -529,9 +511,11 @@ def _on_completion(f): timeout_handle.cancel() if not waiter.done(): waiter.set_result(None) + futures.future_discard_from_awaited_by(f, cur_task) for f in fs: f.add_done_callback(_on_completion) + futures.future_add_to_awaited_by(f, cur_task) try: await waiter @@ -817,10 +801,19 @@ def gather(*coros_or_futures, return_exceptions=False): outer.set_result([]) return outer - def _done_callback(fut): + loop = events._get_running_loop() + if loop is not None: + cur_task = current_task(loop) + else: + cur_task = None + + def _done_callback(fut, cur_task=cur_task): nonlocal nfinished nfinished += 1 + if cur_task is not None: + futures.future_discard_from_awaited_by(fut, cur_task) + if outer is None or outer.done(): if not fut.cancelled(): # Mark exception retrieved. @@ -877,7 +870,6 @@ def _done_callback(fut): nfuts = 0 nfinished = 0 done_futs = [] - loop = None outer = None # bpo-46672 for arg in coros_or_futures: if arg not in arg_to_fut: @@ -890,12 +882,13 @@ def _done_callback(fut): # can't control it, disable the "destroy pending task" # warning. fut._log_destroy_pending = False - nfuts += 1 arg_to_fut[arg] = fut if fut.done(): done_futs.append(fut) else: + if cur_task is not None: + futures.future_add_to_awaited_by(fut, cur_task) fut.add_done_callback(_done_callback) else: @@ -915,6 +908,25 @@ def _done_callback(fut): return outer +def _log_on_exception(fut): + if fut.cancelled(): + return + + exc = fut.exception() + if exc is None: + return + + context = { + 'message': + f'{exc.__class__.__name__} exception in shielded future', + 'exception': exc, + 'future': fut, + } + if fut._source_traceback: + context['source_traceback'] = fut._source_traceback + fut._loop.call_exception_handler(context) + + def shield(arg): """Wait for a future, shielding it from cancellation. @@ -955,11 +967,16 @@ def shield(arg): loop = futures._get_loop(inner) outer = loop.create_future() + if loop is not None and (cur_task := current_task(loop)) is not None: + futures.future_add_to_awaited_by(inner, cur_task) + else: + cur_task = None + + def _clear_awaited_by_callback(inner): + futures.future_discard_from_awaited_by(inner, cur_task) + def _inner_done_callback(inner): if outer.cancelled(): - if not inner.cancelled(): - # Mark inner's result as retrieved. - inner.exception() return if inner.cancelled(): @@ -971,10 +988,16 @@ def _inner_done_callback(inner): else: outer.set_result(inner.result()) - def _outer_done_callback(outer): if not inner.done(): inner.remove_done_callback(_inner_done_callback) + # Keep only one callback to log on cancel + inner.remove_done_callback(_log_on_exception) + inner.add_done_callback(_log_on_exception) + + if cur_task is not None: + inner.add_done_callback(_clear_awaited_by_callback) + inner.add_done_callback(_inner_done_callback) outer.add_done_callback(_outer_done_callback) @@ -1023,9 +1046,9 @@ def create_eager_task_factory(custom_task_constructor): used. E.g. `loop.set_task_factory(asyncio.eager_task_factory)`. """ - def factory(loop, coro, *, name=None, context=None): + def factory(loop, coro, *, eager_start=True, **kwargs): return custom_task_constructor( - coro, loop=loop, name=name, context=context, eager_start=True) + coro, loop=loop, eager_start=eager_start, **kwargs) return factory @@ -1097,14 +1120,13 @@ def _unregister_eager_task(task): _py_enter_task = _enter_task _py_leave_task = _leave_task _py_swap_current_task = _swap_current_task - +_py_all_tasks = all_tasks try: from _asyncio import (_register_task, _register_eager_task, _unregister_task, _unregister_eager_task, _enter_task, _leave_task, _swap_current_task, - _scheduled_tasks, _eager_tasks, _current_tasks, - current_task) + current_task, all_tasks) except ImportError: pass else: @@ -1116,3 +1138,4 @@ def _unregister_eager_task(task): _c_enter_task = _enter_task _c_leave_task = _leave_task _c_swap_current_task = _swap_current_task + _c_all_tasks = all_tasks diff --git a/Lib/asyncio/timeouts.py b/Lib/asyncio/timeouts.py index e6f5100691d..09342dc7c13 100644 --- a/Lib/asyncio/timeouts.py +++ b/Lib/asyncio/timeouts.py @@ -1,7 +1,6 @@ import enum from types import TracebackType -from typing import final, Optional, Type from . import events from . import exceptions @@ -23,14 +22,13 @@ class _State(enum.Enum): EXITED = "finished" -@final class Timeout: """Asynchronous context manager for cancelling overdue coroutines. Use `timeout()` or `timeout_at()` rather than instantiating this class directly. """ - def __init__(self, when: Optional[float]) -> None: + def __init__(self, when: float | None) -> None: """Schedule a timeout that will trigger at a given loop time. - If `when` is `None`, the timeout will never trigger. @@ -39,15 +37,15 @@ def __init__(self, when: Optional[float]) -> None: """ self._state = _State.CREATED - self._timeout_handler: Optional[events.TimerHandle] = None - self._task: Optional[tasks.Task] = None + self._timeout_handler: events.TimerHandle | None = None + self._task: tasks.Task | None = None self._when = when - def when(self) -> Optional[float]: + def when(self) -> float | None: """Return the current deadline.""" return self._when - def reschedule(self, when: Optional[float]) -> None: + def reschedule(self, when: float | None) -> None: """Reschedule the timeout.""" if self._state is not _State.ENTERED: if self._state is _State.CREATED: @@ -96,10 +94,10 @@ async def __aenter__(self) -> "Timeout": async def __aexit__( self, - exc_type: Optional[Type[BaseException]], - exc_val: Optional[BaseException], - exc_tb: Optional[TracebackType], - ) -> Optional[bool]: + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> bool | None: assert self._state in (_State.ENTERED, _State.EXPIRING) if self._timeout_handler is not None: @@ -142,7 +140,7 @@ def _insert_timeout_error(exc_val: BaseException) -> None: exc_val = exc_val.__context__ -def timeout(delay: Optional[float]) -> Timeout: +def timeout(delay: float | None) -> Timeout: """Timeout async context manager. Useful in cases when you want to apply timeout logic around block @@ -162,7 +160,7 @@ def timeout(delay: Optional[float]) -> Timeout: return Timeout(loop.time() + delay if delay is not None else None) -def timeout_at(when: Optional[float]) -> Timeout: +def timeout_at(when: float | None) -> Timeout: """Schedule the timeout at absolute time. Like timeout() but argument gives absolute time in the same clock system diff --git a/Lib/asyncio/tools.py b/Lib/asyncio/tools.py new file mode 100644 index 00000000000..f39e11fdd51 --- /dev/null +++ b/Lib/asyncio/tools.py @@ -0,0 +1,276 @@ +"""Tools to analyze tasks running in asyncio programs.""" + +from collections import defaultdict, namedtuple +from itertools import count +from enum import Enum +import sys +from _remote_debugging import RemoteUnwinder, FrameInfo + +class NodeType(Enum): + COROUTINE = 1 + TASK = 2 + + +class CycleFoundException(Exception): + """Raised when there is a cycle when drawing the call tree.""" + def __init__( + self, + cycles: list[list[int]], + id2name: dict[int, str], + ) -> None: + super().__init__(cycles, id2name) + self.cycles = cycles + self.id2name = id2name + + + +# ─── indexing helpers ─────────────────────────────────────────── +def _format_stack_entry(elem: str|FrameInfo) -> str: + if not isinstance(elem, str): + if elem.lineno == 0 and elem.filename == "": + return f"{elem.funcname}" + else: + return f"{elem.funcname} {elem.filename}:{elem.lineno}" + return elem + + +def _index(result): + id2name, awaits, task_stacks = {}, [], {} + for awaited_info in result: + for task_info in awaited_info.awaited_by: + task_id = task_info.task_id + task_name = task_info.task_name + id2name[task_id] = task_name + + # Store the internal coroutine stack for this task + if task_info.coroutine_stack: + for coro_info in task_info.coroutine_stack: + call_stack = coro_info.call_stack + internal_stack = [_format_stack_entry(frame) for frame in call_stack] + task_stacks[task_id] = internal_stack + + # Add the awaited_by relationships (external dependencies) + if task_info.awaited_by: + for coro_info in task_info.awaited_by: + call_stack = coro_info.call_stack + parent_task_id = coro_info.task_name + stack = [_format_stack_entry(frame) for frame in call_stack] + awaits.append((parent_task_id, stack, task_id)) + return id2name, awaits, task_stacks + + +def _build_tree(id2name, awaits, task_stacks): + id2label = {(NodeType.TASK, tid): name for tid, name in id2name.items()} + children = defaultdict(list) + cor_nodes = defaultdict(dict) # Maps parent -> {frame_name: node_key} + next_cor_id = count(1) + + def get_or_create_cor_node(parent, frame): + """Get existing coroutine node or create new one under parent""" + if frame in cor_nodes[parent]: + return cor_nodes[parent][frame] + + node_key = (NodeType.COROUTINE, f"c{next(next_cor_id)}") + id2label[node_key] = frame + children[parent].append(node_key) + cor_nodes[parent][frame] = node_key + return node_key + + # Build task dependency tree with coroutine frames + for parent_id, stack, child_id in awaits: + cur = (NodeType.TASK, parent_id) + for frame in reversed(stack): + cur = get_or_create_cor_node(cur, frame) + + child_key = (NodeType.TASK, child_id) + if child_key not in children[cur]: + children[cur].append(child_key) + + # Add coroutine stacks for leaf tasks + awaiting_tasks = {parent_id for parent_id, _, _ in awaits} + for task_id in id2name: + if task_id not in awaiting_tasks and task_id in task_stacks: + cur = (NodeType.TASK, task_id) + for frame in reversed(task_stacks[task_id]): + cur = get_or_create_cor_node(cur, frame) + + return id2label, children + + +def _roots(id2label, children): + all_children = {c for kids in children.values() for c in kids} + return [n for n in id2label if n not in all_children] + +# ─── detect cycles in the task-to-task graph ─────────────────────── +def _task_graph(awaits): + """Return {parent_task_id: {child_task_id, …}, …}.""" + g = defaultdict(set) + for parent_id, _stack, child_id in awaits: + g[parent_id].add(child_id) + return g + + +def _find_cycles(graph): + """ + Depth-first search for back-edges. + + Returns a list of cycles (each cycle is a list of task-ids) or an + empty list if the graph is acyclic. + """ + WHITE, GREY, BLACK = 0, 1, 2 + color = defaultdict(lambda: WHITE) + path, cycles = [], [] + + def dfs(v): + color[v] = GREY + path.append(v) + for w in graph.get(v, ()): + if color[w] == WHITE: + dfs(w) + elif color[w] == GREY: # back-edge → cycle! + i = path.index(w) + cycles.append(path[i:] + [w]) # make a copy + color[v] = BLACK + path.pop() + + for v in list(graph): + if color[v] == WHITE: + dfs(v) + return cycles + + +# ─── PRINT TREE FUNCTION ─────────────────────────────────────── +def get_all_awaited_by(pid): + unwinder = RemoteUnwinder(pid) + return unwinder.get_all_awaited_by() + + +def build_async_tree(result, task_emoji="(T)", cor_emoji=""): + """ + Build a list of strings for pretty-print an async call tree. + + The call tree is produced by `get_all_async_stacks()`, prefixing tasks + with `task_emoji` and coroutine frames with `cor_emoji`. + """ + id2name, awaits, task_stacks = _index(result) + g = _task_graph(awaits) + cycles = _find_cycles(g) + if cycles: + raise CycleFoundException(cycles, id2name) + labels, children = _build_tree(id2name, awaits, task_stacks) + + def pretty(node): + flag = task_emoji if node[0] == NodeType.TASK else cor_emoji + return f"{flag} {labels[node]}" + + def render(node, prefix="", last=True, buf=None): + if buf is None: + buf = [] + buf.append(f"{prefix}{'└── ' if last else '├── '}{pretty(node)}") + new_pref = prefix + (" " if last else "│ ") + kids = children.get(node, []) + for i, kid in enumerate(kids): + render(kid, new_pref, i == len(kids) - 1, buf) + return buf + + return [render(root) for root in _roots(labels, children)] + + +def build_task_table(result): + id2name, _, _ = _index(result) + table = [] + + for awaited_info in result: + thread_id = awaited_info.thread_id + for task_info in awaited_info.awaited_by: + # Get task info + task_id = task_info.task_id + task_name = task_info.task_name + + # Build coroutine stack string + frames = [frame for coro in task_info.coroutine_stack + for frame in coro.call_stack] + coro_stack = " -> ".join(_format_stack_entry(x).split(" ")[0] + for x in frames) + + # Handle tasks with no awaiters + if not task_info.awaited_by: + table.append([thread_id, hex(task_id), task_name, coro_stack, + "", "", "0x0"]) + continue + + # Handle tasks with awaiters + for coro_info in task_info.awaited_by: + parent_id = coro_info.task_name + awaiter_frames = [_format_stack_entry(x).split(" ")[0] + for x in coro_info.call_stack] + awaiter_chain = " -> ".join(awaiter_frames) + awaiter_name = id2name.get(parent_id, "Unknown") + parent_id_str = (hex(parent_id) if isinstance(parent_id, int) + else str(parent_id)) + + table.append([thread_id, hex(task_id), task_name, coro_stack, + awaiter_chain, awaiter_name, parent_id_str]) + + return table + +def _print_cycle_exception(exception: CycleFoundException): + print("ERROR: await-graph contains cycles - cannot print a tree!", file=sys.stderr) + print("", file=sys.stderr) + for c in exception.cycles: + inames = " → ".join(exception.id2name.get(tid, hex(tid)) for tid in c) + print(f"cycle: {inames}", file=sys.stderr) + + +def exit_with_permission_help_text(): + """ + Prints a message pointing to platform-specific permission help text and exits the program. + This function is called when a PermissionError is encountered while trying + to attach to a process. + """ + print( + "Error: The specified process cannot be attached to due to insufficient permissions.\n" + "See the Python documentation for details on required privileges and troubleshooting:\n" + "https://docs.python.org/3.14/howto/remote_debugging.html#permission-requirements\n" + ) + sys.exit(1) + + +def _get_awaited_by_tasks(pid: int) -> list: + try: + return get_all_awaited_by(pid) + except RuntimeError as e: + while e.__context__ is not None: + e = e.__context__ + print(f"Error retrieving tasks: {e}") + sys.exit(1) + except PermissionError as e: + exit_with_permission_help_text() + + +def display_awaited_by_tasks_table(pid: int) -> None: + """Build and print a table of all pending tasks under `pid`.""" + + tasks = _get_awaited_by_tasks(pid) + table = build_task_table(tasks) + # Print the table in a simple tabular format + print( + f"{'tid':<10} {'task id':<20} {'task name':<20} {'coroutine stack':<50} {'awaiter chain':<50} {'awaiter name':<15} {'awaiter id':<15}" + ) + print("-" * 180) + for row in table: + print(f"{row[0]:<10} {row[1]:<20} {row[2]:<20} {row[3]:<50} {row[4]:<50} {row[5]:<15} {row[6]:<15}") + + +def display_awaited_by_tasks_tree(pid: int) -> None: + """Build and print a tree of all pending tasks under `pid`.""" + + tasks = _get_awaited_by_tasks(pid) + try: + result = build_async_tree(tasks) + except CycleFoundException as e: + _print_cycle_exception(e) + sys.exit(1) + + for tree in result: + print("\n".join(tree)) diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py index 41ccf1b78fb..1c1458127db 100644 --- a/Lib/asyncio/unix_events.py +++ b/Lib/asyncio/unix_events.py @@ -28,10 +28,6 @@ __all__ = ( 'SelectorEventLoop', - 'AbstractChildWatcher', 'SafeChildWatcher', - 'FastChildWatcher', 'PidfdChildWatcher', - 'MultiLoopChildWatcher', 'ThreadedChildWatcher', - 'DefaultEventLoopPolicy', 'EventLoop', ) @@ -65,6 +61,10 @@ def __init__(self, selector=None): super().__init__(selector) self._signal_handlers = {} self._unix_server_sockets = {} + if can_use_pidfd(): + self._watcher = _PidfdChildWatcher() + else: + self._watcher = _ThreadedChildWatcher() def close(self): super().close() @@ -94,7 +94,7 @@ def add_signal_handler(self, sig, callback, *args): Raise RuntimeError if there is a problem setting up the handler. """ if (coroutines.iscoroutine(callback) or - coroutines.iscoroutinefunction(callback)): + coroutines._iscoroutinefunction(callback)): raise TypeError("coroutines cannot be used " "with add_signal_handler()") self._check_signal(sig) @@ -197,33 +197,22 @@ def _make_write_pipe_transport(self, pipe, protocol, waiter=None, async def _make_subprocess_transport(self, protocol, args, shell, stdin, stdout, stderr, bufsize, extra=None, **kwargs): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - watcher = events.get_child_watcher() - - with watcher: - if not watcher.is_active(): - # Check early. - # Raising exception before process creation - # prevents subprocess execution if the watcher - # is not ready to handle it. - raise RuntimeError("asyncio.get_child_watcher() is not activated, " - "subprocess support is not installed.") - waiter = self.create_future() - transp = _UnixSubprocessTransport(self, protocol, args, shell, - stdin, stdout, stderr, bufsize, - waiter=waiter, extra=extra, - **kwargs) - watcher.add_child_handler(transp.get_pid(), - self._child_watcher_callback, transp) - try: - await waiter - except (SystemExit, KeyboardInterrupt): - raise - except BaseException: - transp.close() - await transp._wait() - raise + watcher = self._watcher + waiter = self.create_future() + transp = _UnixSubprocessTransport(self, protocol, args, shell, + stdin, stdout, stderr, bufsize, + waiter=waiter, extra=extra, + **kwargs) + watcher.add_child_handler(transp.get_pid(), + self._child_watcher_callback, transp) + try: + await waiter + except (SystemExit, KeyboardInterrupt): + raise + except BaseException: + transp.close() + await transp._wait() + raise return transp @@ -865,93 +854,7 @@ def _start(self, args, shell, stdin, stdout, stderr, bufsize, **kwargs): stdin_w.close() -class AbstractChildWatcher: - """Abstract base class for monitoring child processes. - - Objects derived from this class monitor a collection of subprocesses and - report their termination or interruption by a signal. - - New callbacks are registered with .add_child_handler(). Starting a new - process must be done within a 'with' block to allow the watcher to suspend - its activity until the new process if fully registered (this is needed to - prevent a race condition in some implementations). - - Example: - with watcher: - proc = subprocess.Popen("sleep 1") - watcher.add_child_handler(proc.pid, callback) - - Notes: - Implementations of this class must be thread-safe. - - Since child watcher objects may catch the SIGCHLD signal and call - waitpid(-1), there should be only one active object per process. - """ - - def __init_subclass__(cls) -> None: - if cls.__module__ != __name__: - warnings._deprecated("AbstractChildWatcher", - "{name!r} is deprecated as of Python 3.12 and will be " - "removed in Python {remove}.", - remove=(3, 14)) - - def add_child_handler(self, pid, callback, *args): - """Register a new child handler. - - Arrange for callback(pid, returncode, *args) to be called when - process 'pid' terminates. Specifying another callback for the same - process replaces the previous handler. - - Note: callback() must be thread-safe. - """ - raise NotImplementedError() - - def remove_child_handler(self, pid): - """Removes the handler for process 'pid'. - - The function returns True if the handler was successfully removed, - False if there was nothing to remove.""" - - raise NotImplementedError() - - def attach_loop(self, loop): - """Attach the watcher to an event loop. - - If the watcher was previously attached to an event loop, then it is - first detached before attaching to the new loop. - - Note: loop may be None. - """ - raise NotImplementedError() - - def close(self): - """Close the watcher. - - This must be called to make sure that any underlying resource is freed. - """ - raise NotImplementedError() - - def is_active(self): - """Return ``True`` if the watcher is active and is used by the event loop. - - Return True if the watcher is installed and ready to handle process exit - notifications. - - """ - raise NotImplementedError() - - def __enter__(self): - """Enter the watcher's context and allow starting new processes - - This function must return self""" - raise NotImplementedError() - - def __exit__(self, a, b, c): - """Exit the watcher's context""" - raise NotImplementedError() - - -class PidfdChildWatcher(AbstractChildWatcher): +class _PidfdChildWatcher: """Child watcher implementation using Linux's pid file descriptors. This child watcher polls process file descriptors (pidfds) to await child @@ -963,21 +866,6 @@ class PidfdChildWatcher(AbstractChildWatcher): recent (5.3+) kernels. """ - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_value, exc_traceback): - pass - - def is_active(self): - return True - - def close(self): - pass - - def attach_loop(self, loop): - pass - def add_child_handler(self, pid, callback, *args): loop = events.get_running_loop() pidfd = os.pidfd_open(pid) @@ -1002,386 +890,7 @@ def _do_wait(self, pid, pidfd, callback, args): os.close(pidfd) callback(pid, returncode, *args) - def remove_child_handler(self, pid): - # asyncio never calls remove_child_handler() !!! - # The method is no-op but is implemented because - # abstract base classes require it. - return True - - -class BaseChildWatcher(AbstractChildWatcher): - - def __init__(self): - self._loop = None - self._callbacks = {} - - def close(self): - self.attach_loop(None) - - def is_active(self): - return self._loop is not None and self._loop.is_running() - - def _do_waitpid(self, expected_pid): - raise NotImplementedError() - - def _do_waitpid_all(self): - raise NotImplementedError() - - def attach_loop(self, loop): - assert loop is None or isinstance(loop, events.AbstractEventLoop) - - if self._loop is not None and loop is None and self._callbacks: - warnings.warn( - 'A loop is being detached ' - 'from a child watcher with pending handlers', - RuntimeWarning) - - if self._loop is not None: - self._loop.remove_signal_handler(signal.SIGCHLD) - - self._loop = loop - if loop is not None: - loop.add_signal_handler(signal.SIGCHLD, self._sig_chld) - - # Prevent a race condition in case a child terminated - # during the switch. - self._do_waitpid_all() - - def _sig_chld(self): - try: - self._do_waitpid_all() - except (SystemExit, KeyboardInterrupt): - raise - except BaseException as exc: - # self._loop should always be available here - # as '_sig_chld' is added as a signal handler - # in 'attach_loop' - self._loop.call_exception_handler({ - 'message': 'Unknown exception in SIGCHLD handler', - 'exception': exc, - }) - - -class SafeChildWatcher(BaseChildWatcher): - """'Safe' child watcher implementation. - - This implementation avoids disrupting other code spawning processes by - polling explicitly each process in the SIGCHLD handler instead of calling - os.waitpid(-1). - - This is a safe solution but it has a significant overhead when handling a - big number of children (O(n) each time SIGCHLD is raised) - """ - - def __init__(self): - super().__init__() - warnings._deprecated("SafeChildWatcher", - "{name!r} is deprecated as of Python 3.12 and will be " - "removed in Python {remove}.", - remove=(3, 14)) - - def close(self): - self._callbacks.clear() - super().close() - - def __enter__(self): - return self - - def __exit__(self, a, b, c): - pass - - def add_child_handler(self, pid, callback, *args): - self._callbacks[pid] = (callback, args) - - # Prevent a race condition in case the child is already terminated. - self._do_waitpid(pid) - - def remove_child_handler(self, pid): - try: - del self._callbacks[pid] - return True - except KeyError: - return False - - def _do_waitpid_all(self): - - for pid in list(self._callbacks): - self._do_waitpid(pid) - - def _do_waitpid(self, expected_pid): - assert expected_pid > 0 - - try: - pid, status = os.waitpid(expected_pid, os.WNOHANG) - except ChildProcessError: - # The child process is already reaped - # (may happen if waitpid() is called elsewhere). - pid = expected_pid - returncode = 255 - logger.warning( - "Unknown child process pid %d, will report returncode 255", - pid) - else: - if pid == 0: - # The child process is still alive. - return - - returncode = waitstatus_to_exitcode(status) - if self._loop.get_debug(): - logger.debug('process %s exited with returncode %s', - expected_pid, returncode) - - try: - callback, args = self._callbacks.pop(pid) - except KeyError: # pragma: no cover - # May happen if .remove_child_handler() is called - # after os.waitpid() returns. - if self._loop.get_debug(): - logger.warning("Child watcher got an unexpected pid: %r", - pid, exc_info=True) - else: - callback(pid, returncode, *args) - - -class FastChildWatcher(BaseChildWatcher): - """'Fast' child watcher implementation. - - This implementation reaps every terminated processes by calling - os.waitpid(-1) directly, possibly breaking other code spawning processes - and waiting for their termination. - - There is no noticeable overhead when handling a big number of children - (O(1) each time a child terminates). - """ - def __init__(self): - super().__init__() - self._lock = threading.Lock() - self._zombies = {} - self._forks = 0 - warnings._deprecated("FastChildWatcher", - "{name!r} is deprecated as of Python 3.12 and will be " - "removed in Python {remove}.", - remove=(3, 14)) - - def close(self): - self._callbacks.clear() - self._zombies.clear() - super().close() - - def __enter__(self): - with self._lock: - self._forks += 1 - - return self - - def __exit__(self, a, b, c): - with self._lock: - self._forks -= 1 - - if self._forks or not self._zombies: - return - - collateral_victims = str(self._zombies) - self._zombies.clear() - - logger.warning( - "Caught subprocesses termination from unknown pids: %s", - collateral_victims) - - def add_child_handler(self, pid, callback, *args): - assert self._forks, "Must use the context manager" - - with self._lock: - try: - returncode = self._zombies.pop(pid) - except KeyError: - # The child is running. - self._callbacks[pid] = callback, args - return - - # The child is dead already. We can fire the callback. - callback(pid, returncode, *args) - - def remove_child_handler(self, pid): - try: - del self._callbacks[pid] - return True - except KeyError: - return False - - def _do_waitpid_all(self): - # Because of signal coalescing, we must keep calling waitpid() as - # long as we're able to reap a child. - while True: - try: - pid, status = os.waitpid(-1, os.WNOHANG) - except ChildProcessError: - # No more child processes exist. - return - else: - if pid == 0: - # A child process is still alive. - return - - returncode = waitstatus_to_exitcode(status) - - with self._lock: - try: - callback, args = self._callbacks.pop(pid) - except KeyError: - # unknown child - if self._forks: - # It may not be registered yet. - self._zombies[pid] = returncode - if self._loop.get_debug(): - logger.debug('unknown process %s exited ' - 'with returncode %s', - pid, returncode) - continue - callback = None - else: - if self._loop.get_debug(): - logger.debug('process %s exited with returncode %s', - pid, returncode) - - if callback is None: - logger.warning( - "Caught subprocess termination from unknown pid: " - "%d -> %d", pid, returncode) - else: - callback(pid, returncode, *args) - - -class MultiLoopChildWatcher(AbstractChildWatcher): - """A watcher that doesn't require running loop in the main thread. - - This implementation registers a SIGCHLD signal handler on - instantiation (which may conflict with other code that - install own handler for this signal). - - The solution is safe but it has a significant overhead when - handling a big number of processes (*O(n)* each time a - SIGCHLD is received). - """ - - # Implementation note: - # The class keeps compatibility with AbstractChildWatcher ABC - # To achieve this it has empty attach_loop() method - # and doesn't accept explicit loop argument - # for add_child_handler()/remove_child_handler() - # but retrieves the current loop by get_running_loop() - - def __init__(self): - self._callbacks = {} - self._saved_sighandler = None - warnings._deprecated("MultiLoopChildWatcher", - "{name!r} is deprecated as of Python 3.12 and will be " - "removed in Python {remove}.", - remove=(3, 14)) - - def is_active(self): - return self._saved_sighandler is not None - - def close(self): - self._callbacks.clear() - if self._saved_sighandler is None: - return - - handler = signal.getsignal(signal.SIGCHLD) - if handler != self._sig_chld: - logger.warning("SIGCHLD handler was changed by outside code") - else: - signal.signal(signal.SIGCHLD, self._saved_sighandler) - self._saved_sighandler = None - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - pass - - def add_child_handler(self, pid, callback, *args): - loop = events.get_running_loop() - self._callbacks[pid] = (loop, callback, args) - - # Prevent a race condition in case the child is already terminated. - self._do_waitpid(pid) - - def remove_child_handler(self, pid): - try: - del self._callbacks[pid] - return True - except KeyError: - return False - - def attach_loop(self, loop): - # Don't save the loop but initialize itself if called first time - # The reason to do it here is that attach_loop() is called from - # unix policy only for the main thread. - # Main thread is required for subscription on SIGCHLD signal - if self._saved_sighandler is not None: - return - - self._saved_sighandler = signal.signal(signal.SIGCHLD, self._sig_chld) - if self._saved_sighandler is None: - logger.warning("Previous SIGCHLD handler was set by non-Python code, " - "restore to default handler on watcher close.") - self._saved_sighandler = signal.SIG_DFL - - # Set SA_RESTART to limit EINTR occurrences. - signal.siginterrupt(signal.SIGCHLD, False) - - def _do_waitpid_all(self): - for pid in list(self._callbacks): - self._do_waitpid(pid) - - def _do_waitpid(self, expected_pid): - assert expected_pid > 0 - - try: - pid, status = os.waitpid(expected_pid, os.WNOHANG) - except ChildProcessError: - # The child process is already reaped - # (may happen if waitpid() is called elsewhere). - pid = expected_pid - returncode = 255 - logger.warning( - "Unknown child process pid %d, will report returncode 255", - pid) - debug_log = False - else: - if pid == 0: - # The child process is still alive. - return - - returncode = waitstatus_to_exitcode(status) - debug_log = True - try: - loop, callback, args = self._callbacks.pop(pid) - except KeyError: # pragma: no cover - # May happen if .remove_child_handler() is called - # after os.waitpid() returns. - logger.warning("Child watcher got an unexpected pid: %r", - pid, exc_info=True) - else: - if loop.is_closed(): - logger.warning("Loop %r that handles pid %r is closed", loop, pid) - else: - if debug_log and loop.get_debug(): - logger.debug('process %s exited with returncode %s', - expected_pid, returncode) - loop.call_soon_threadsafe(callback, pid, returncode, *args) - - def _sig_chld(self, signum, frame): - try: - self._do_waitpid_all() - except (SystemExit, KeyboardInterrupt): - raise - except BaseException: - logger.warning('Unknown exception in SIGCHLD handler', exc_info=True) - - -class ThreadedChildWatcher(AbstractChildWatcher): +class _ThreadedChildWatcher: """Threaded child watcher implementation. The watcher uses a thread per process @@ -1398,18 +907,6 @@ def __init__(self): self._pid_counter = itertools.count(0) self._threads = {} - def is_active(self): - return True - - def close(self): - pass - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - pass - def __del__(self, _warn=warnings.warn): threads = [thread for thread in list(self._threads.values()) if thread.is_alive()] @@ -1427,15 +924,6 @@ def add_child_handler(self, pid, callback, *args): self._threads[pid] = thread thread.start() - def remove_child_handler(self, pid): - # asyncio never calls remove_child_handler() !!! - # The method is no-op but is implemented because - # abstract base classes require it. - return True - - def attach_loop(self, loop): - pass - def _do_waitpid(self, loop, expected_pid, callback, args): assert expected_pid > 0 @@ -1474,63 +962,11 @@ def can_use_pidfd(): return True -class _UnixDefaultEventLoopPolicy(events.BaseDefaultEventLoopPolicy): - """UNIX event loop policy with a watcher for child processes.""" +class _UnixDefaultEventLoopPolicy(events._BaseDefaultEventLoopPolicy): + """UNIX event loop policy""" _loop_factory = _UnixSelectorEventLoop - def __init__(self): - super().__init__() - self._watcher = None - - def _init_watcher(self): - with events._lock: - if self._watcher is None: # pragma: no branch - if can_use_pidfd(): - self._watcher = PidfdChildWatcher() - else: - self._watcher = ThreadedChildWatcher() - - def set_event_loop(self, loop): - """Set the event loop. - - As a side effect, if a child watcher was set before, then calling - .set_event_loop() from the main thread will call .attach_loop(loop) on - the child watcher. - """ - - super().set_event_loop(loop) - - if (self._watcher is not None and - threading.current_thread() is threading.main_thread()): - self._watcher.attach_loop(loop) - - def get_child_watcher(self): - """Get the watcher for child processes. - - If not yet set, a ThreadedChildWatcher object is automatically created. - """ - if self._watcher is None: - self._init_watcher() - - warnings._deprecated("get_child_watcher", - "{name!r} is deprecated as of Python 3.12 and will be " - "removed in Python {remove}.", remove=(3, 14)) - return self._watcher - - def set_child_watcher(self, watcher): - """Set the watcher for child processes.""" - - assert watcher is None or isinstance(watcher, AbstractChildWatcher) - - if self._watcher is not None: - self._watcher.close() - - self._watcher = watcher - warnings._deprecated("set_child_watcher", - "{name!r} is deprecated as of Python 3.12 and will be " - "removed in Python {remove}.", remove=(3, 14)) - SelectorEventLoop = _UnixSelectorEventLoop -DefaultEventLoopPolicy = _UnixDefaultEventLoopPolicy +_DefaultEventLoopPolicy = _UnixDefaultEventLoopPolicy EventLoop = SelectorEventLoop diff --git a/Lib/asyncio/windows_events.py b/Lib/asyncio/windows_events.py index bf99bc271c7..5f75b17d8ca 100644 --- a/Lib/asyncio/windows_events.py +++ b/Lib/asyncio/windows_events.py @@ -29,8 +29,8 @@ __all__ = ( 'SelectorEventLoop', 'ProactorEventLoop', 'IocpProactor', - 'DefaultEventLoopPolicy', 'WindowsSelectorEventLoopPolicy', - 'WindowsProactorEventLoopPolicy', 'EventLoop', + '_DefaultEventLoopPolicy', '_WindowsSelectorEventLoopPolicy', + '_WindowsProactorEventLoopPolicy', 'EventLoop', ) @@ -891,13 +891,13 @@ def callback(f): SelectorEventLoop = _WindowsSelectorEventLoop -class WindowsSelectorEventLoopPolicy(events.BaseDefaultEventLoopPolicy): +class _WindowsSelectorEventLoopPolicy(events._BaseDefaultEventLoopPolicy): _loop_factory = SelectorEventLoop -class WindowsProactorEventLoopPolicy(events.BaseDefaultEventLoopPolicy): +class _WindowsProactorEventLoopPolicy(events._BaseDefaultEventLoopPolicy): _loop_factory = ProactorEventLoop -DefaultEventLoopPolicy = WindowsProactorEventLoopPolicy +_DefaultEventLoopPolicy = _WindowsProactorEventLoopPolicy EventLoop = ProactorEventLoop diff --git a/Lib/bdb.py b/Lib/bdb.py index f256b56daaa..79da4bab9c9 100644 --- a/Lib/bdb.py +++ b/Lib/bdb.py @@ -2,7 +2,9 @@ import fnmatch import sys +import threading import os +import weakref from contextlib import contextmanager from inspect import CO_GENERATOR, CO_COROUTINE, CO_ASYNC_GENERATOR @@ -15,6 +17,166 @@ class BdbQuit(Exception): """Exception to give up completely.""" +E = sys.monitoring.events + +class _MonitoringTracer: + EVENT_CALLBACK_MAP = { + E.PY_START: 'call', + E.PY_RESUME: 'call', + E.PY_THROW: 'call', + E.LINE: 'line', + E.JUMP: 'jump', + E.PY_RETURN: 'return', + E.PY_YIELD: 'return', + E.PY_UNWIND: 'unwind', + E.RAISE: 'exception', + E.STOP_ITERATION: 'exception', + E.INSTRUCTION: 'opcode', + } + + GLOBAL_EVENTS = E.PY_START | E.PY_RESUME | E.PY_THROW | E.PY_UNWIND | E.RAISE + LOCAL_EVENTS = E.LINE | E.JUMP | E.PY_RETURN | E.PY_YIELD | E.STOP_ITERATION + + def __init__(self): + self._tool_id = sys.monitoring.DEBUGGER_ID + self._name = 'bdbtracer' + self._tracefunc = None + self._disable_current_event = False + self._tracing_thread = None + self._enabled = False + + def start_trace(self, tracefunc): + self._tracefunc = tracefunc + self._tracing_thread = threading.current_thread() + curr_tool = sys.monitoring.get_tool(self._tool_id) + if curr_tool is None: + sys.monitoring.use_tool_id(self._tool_id, self._name) + elif curr_tool == self._name: + sys.monitoring.clear_tool_id(self._tool_id) + else: + raise ValueError('Another debugger is using the monitoring tool') + E = sys.monitoring.events + all_events = 0 + for event, cb_name in self.EVENT_CALLBACK_MAP.items(): + callback = self.callback_wrapper(getattr(self, f'{cb_name}_callback'), event) + sys.monitoring.register_callback(self._tool_id, event, callback) + if event != E.INSTRUCTION: + all_events |= event + self.update_local_events() + sys.monitoring.set_events(self._tool_id, self.GLOBAL_EVENTS) + self._enabled = True + + def stop_trace(self): + self._enabled = False + self._tracing_thread = None + curr_tool = sys.monitoring.get_tool(self._tool_id) + if curr_tool != self._name: + return + sys.monitoring.clear_tool_id(self._tool_id) + sys.monitoring.free_tool_id(self._tool_id) + + def disable_current_event(self): + self._disable_current_event = True + + def restart_events(self): + if sys.monitoring.get_tool(self._tool_id) == self._name: + sys.monitoring.restart_events() + + def callback_wrapper(self, func, event): + import functools + + @functools.wraps(func) + def wrapper(*args): + if self._tracing_thread != threading.current_thread(): + return + try: + frame = sys._getframe().f_back + ret = func(frame, *args) + if self._enabled and frame.f_trace: + self.update_local_events() + if ( + self._disable_current_event + and event not in (E.PY_THROW, E.PY_UNWIND, E.RAISE) + ): + return sys.monitoring.DISABLE + else: + return ret + except BaseException: + self.stop_trace() + sys._getframe().f_back.f_trace = None + raise + finally: + self._disable_current_event = False + + return wrapper + + def call_callback(self, frame, code, *args): + local_tracefunc = self._tracefunc(frame, 'call', None) + if local_tracefunc is not None: + frame.f_trace = local_tracefunc + if self._enabled: + sys.monitoring.set_local_events(self._tool_id, code, self.LOCAL_EVENTS) + + def return_callback(self, frame, code, offset, retval): + if frame.f_trace: + frame.f_trace(frame, 'return', retval) + + def unwind_callback(self, frame, code, *args): + if frame.f_trace: + frame.f_trace(frame, 'return', None) + + def line_callback(self, frame, code, *args): + if frame.f_trace and frame.f_trace_lines: + frame.f_trace(frame, 'line', None) + + def jump_callback(self, frame, code, inst_offset, dest_offset): + if dest_offset > inst_offset: + return sys.monitoring.DISABLE + inst_lineno = self._get_lineno(code, inst_offset) + dest_lineno = self._get_lineno(code, dest_offset) + if inst_lineno != dest_lineno: + return sys.monitoring.DISABLE + if frame.f_trace and frame.f_trace_lines: + frame.f_trace(frame, 'line', None) + + def exception_callback(self, frame, code, offset, exc): + if frame.f_trace: + if exc.__traceback__ and hasattr(exc.__traceback__, 'tb_frame'): + tb = exc.__traceback__ + while tb: + if tb.tb_frame.f_locals.get('self') is self: + return + tb = tb.tb_next + frame.f_trace(frame, 'exception', (type(exc), exc, exc.__traceback__)) + + def opcode_callback(self, frame, code, offset): + if frame.f_trace and frame.f_trace_opcodes: + frame.f_trace(frame, 'opcode', None) + + def update_local_events(self, frame=None): + if sys.monitoring.get_tool(self._tool_id) != self._name: + return + if frame is None: + frame = sys._getframe().f_back + while frame is not None: + if frame.f_trace is not None: + if frame.f_trace_opcodes: + events = self.LOCAL_EVENTS | E.INSTRUCTION + else: + events = self.LOCAL_EVENTS + sys.monitoring.set_local_events(self._tool_id, frame.f_code, events) + frame = frame.f_back + + def _get_lineno(self, code, offset): + import dis + last_lineno = None + for start, lineno in dis.findlinestarts(code): + if offset < start: + return last_lineno + last_lineno = lineno + return last_lineno + + class Bdb: """Generic Python debugger base class. @@ -29,7 +191,7 @@ class Bdb: is determined by the __name__ in the frame globals. """ - def __init__(self, skip=None): + def __init__(self, skip=None, backend='settrace'): self.skip = set(skip) if skip else None self.breaks = {} self.fncache = {} @@ -39,6 +201,14 @@ def __init__(self, skip=None): self.enterframe = None self.cmdframe = None self.cmdlineno = None + self.code_linenos = weakref.WeakKeyDictionary() + self.backend = backend + if backend == 'monitoring': + self.monitoring_tracer = _MonitoringTracer() + elif backend == 'settrace': + self.monitoring_tracer = None + else: + raise ValueError(f"Invalid backend '{backend}'") self._load_breaks() @@ -59,6 +229,18 @@ def canonic(self, filename): self.fncache[filename] = canonic return canonic + def start_trace(self): + if self.monitoring_tracer: + self.monitoring_tracer.start_trace(self.trace_dispatch) + else: + sys.settrace(self.trace_dispatch) + + def stop_trace(self): + if self.monitoring_tracer: + self.monitoring_tracer.stop_trace() + else: + sys.settrace(None) + def reset(self): """Set values of attributes as ready to start debugging.""" import linecache @@ -133,7 +315,10 @@ def dispatch_line(self, frame): self.cmdframe == frame and self.cmdlineno == frame.f_lineno ): self.user_line(frame) + self.restart_events() if self.quitting: raise BdbQuit + elif not self.get_break(frame.f_code.co_filename, frame.f_lineno): + self.disable_current_event() return self.trace_dispatch def dispatch_call(self, frame, arg): @@ -149,12 +334,18 @@ def dispatch_call(self, frame, arg): self.botframe = frame.f_back # (CT) Note that this may also be None! return self.trace_dispatch if not (self.stop_here(frame) or self.break_anywhere(frame)): - # No need to trace this function + # We already know there's no breakpoint in this function + # If it's a next/until/return command, we don't need any CALL event + # and we don't need to set the f_trace on any new frame. + # If it's a step command, it must either hit stop_here, or skip the + # whole module. Either way, we don't need the CALL event here. + self.disable_current_event() return # None # Ignore call events in generator except when stepping. if self.stopframe and frame.f_code.co_flags & GENERATOR_AND_COROUTINE_FLAGS: return self.trace_dispatch self.user_call(frame, arg) + self.restart_events() if self.quitting: raise BdbQuit return self.trace_dispatch @@ -168,10 +359,14 @@ def dispatch_return(self, frame, arg): if self.stop_here(frame) or frame == self.returnframe: # Ignore return events in generator except when stepping. if self.stopframe and frame.f_code.co_flags & GENERATOR_AND_COROUTINE_FLAGS: + # It's possible to trigger a StopIteration exception in + # the caller so we must set the trace function in the caller + self._set_caller_tracefunc(frame) return self.trace_dispatch try: self.frame_returning = frame self.user_return(frame, arg) + self.restart_events() finally: self.frame_returning = None if self.quitting: raise BdbQuit @@ -199,6 +394,7 @@ def dispatch_exception(self, frame, arg): if not (frame.f_code.co_flags & GENERATOR_AND_COROUTINE_FLAGS and arg[0] is StopIteration and arg[2] is None): self.user_exception(frame, arg) + self.restart_events() if self.quitting: raise BdbQuit # Stop at the StopIteration or GeneratorExit exception when the user # has set stopframe in a generator by issuing a return command, or a @@ -208,6 +404,7 @@ def dispatch_exception(self, frame, arg): and self.stopframe.f_code.co_flags & GENERATOR_AND_COROUTINE_FLAGS and arg[0] in (StopIteration, GeneratorExit)): self.user_exception(frame, arg) + self.restart_events() if self.quitting: raise BdbQuit return self.trace_dispatch @@ -217,10 +414,14 @@ def dispatch_opcode(self, frame, arg): If the debugger stops on the current opcode, invoke self.user_opcode(). Raise BdbQuit if self.quitting is set. Return self.trace_dispatch to continue tracing in this scope. + + Opcode event will always trigger the user callback. For now the only + opcode event is from an inline set_trace() and we want to stop there + unconditionally. """ - if self.stop_here(frame) or self.break_here(frame): - self.user_opcode(frame) - if self.quitting: raise BdbQuit + self.user_opcode(frame) + self.restart_events() + if self.quitting: raise BdbQuit return self.trace_dispatch # Normally derived classes don't override the following @@ -286,9 +487,25 @@ def do_clear(self, arg): raise NotImplementedError("subclass of bdb must implement do_clear()") def break_anywhere(self, frame): - """Return True if there is any breakpoint for frame's filename. + """Return True if there is any breakpoint in that frame + """ + filename = self.canonic(frame.f_code.co_filename) + if filename not in self.breaks: + return False + for lineno in self.breaks[filename]: + if self._lineno_in_frame(lineno, frame): + return True + return False + + def _lineno_in_frame(self, lineno, frame): + """Return True if the line number is in the frame's code object. """ - return self.canonic(frame.f_code.co_filename) in self.breaks + code = frame.f_code + if lineno < code.co_firstlineno: + return False + if code not in self.code_linenos: + self.code_linenos[code] = set(lineno for _, _, lineno in code.co_lines()) + return lineno in self.code_linenos[code] # Derived classes should override the user_* methods # to gain control. @@ -322,6 +539,8 @@ def _set_trace_opcodes(self, trace_opcodes): if frame is self.botframe: break frame = frame.f_back + if self.monitoring_tracer: + self.monitoring_tracer.update_local_events() def _set_stopinfo(self, stopframe, returnframe, stoplineno=0, opcode=False, cmdframe=None, cmdlineno=None): @@ -381,7 +600,7 @@ def set_next(self, frame): def set_return(self, frame): """Stop when returning from the given frame.""" if frame.f_code.co_flags & GENERATOR_AND_COROUTINE_FLAGS: - self._set_stopinfo(frame, None, -1) + self._set_stopinfo(frame, frame, -1) else: self._set_stopinfo(frame.f_back, frame) @@ -390,6 +609,7 @@ def set_trace(self, frame=None): If frame is not specified, debugging starts from caller's frame. """ + self.stop_trace() if frame is None: frame = sys._getframe().f_back self.reset() @@ -402,7 +622,8 @@ def set_trace(self, frame=None): frame.f_trace_lines = True frame = frame.f_back self.set_stepinstr() - sys.settrace(self.trace_dispatch) + self.enterframe = None + self.start_trace() def set_continue(self): """Stop only at breakpoints or when finished. @@ -413,13 +634,15 @@ def set_continue(self): self._set_stopinfo(self.botframe, None, -1) if not self.breaks: # no breakpoints; run without debugger overhead - sys.settrace(None) + self.stop_trace() frame = sys._getframe().f_back while frame and frame is not self.botframe: del frame.f_trace frame = frame.f_back for frame, (trace_lines, trace_opcodes) in self.frame_trace_lines_opcodes.items(): frame.f_trace_lines, frame.f_trace_opcodes = trace_lines, trace_opcodes + if self.backend == 'monitoring': + self.monitoring_tracer.update_local_events() self.frame_trace_lines_opcodes = {} def set_quit(self): @@ -430,7 +653,7 @@ def set_quit(self): self.stopframe = self.botframe self.returnframe = None self.quitting = True - sys.settrace(None) + self.stop_trace() # Derived classes and clients can call the following methods # to manipulate breakpoints. These methods return an @@ -658,6 +881,16 @@ def format_stack_entry(self, frame_lineno, lprefix=': '): s += f'{lprefix}Warning: lineno is None' return s + def disable_current_event(self): + """Disable the current event.""" + if self.backend == 'monitoring': + self.monitoring_tracer.disable_current_event() + + def restart_events(self): + """Restart all events.""" + if self.backend == 'monitoring': + self.monitoring_tracer.restart_events() + # The following methods can be called by clients to use # a debugger to debug a statement or an expression. # Both can be given as a string, or a code object. @@ -675,14 +908,14 @@ def run(self, cmd, globals=None, locals=None): self.reset() if isinstance(cmd, str): cmd = compile(cmd, "", "exec") - sys.settrace(self.trace_dispatch) + self.start_trace() try: exec(cmd, globals, locals) except BdbQuit: pass finally: self.quitting = True - sys.settrace(None) + self.stop_trace() def runeval(self, expr, globals=None, locals=None): """Debug an expression executed via the eval() function. @@ -695,14 +928,14 @@ def runeval(self, expr, globals=None, locals=None): if locals is None: locals = globals self.reset() - sys.settrace(self.trace_dispatch) + self.start_trace() try: return eval(expr, globals, locals) except BdbQuit: pass finally: self.quitting = True - sys.settrace(None) + self.stop_trace() def runctx(self, cmd, globals, locals): """For backwards-compatibility. Defers to run().""" @@ -717,7 +950,7 @@ def runcall(self, func, /, *args, **kwds): Return the result of the function call. """ self.reset() - sys.settrace(self.trace_dispatch) + self.start_trace() res = None try: res = func(*args, **kwds) @@ -725,7 +958,7 @@ def runcall(self, func, /, *args, **kwds): pass finally: self.quitting = True - sys.settrace(None) + self.stop_trace() return res diff --git a/Lib/code.py b/Lib/code.py index 2777c311187..b134886dc26 100644 --- a/Lib/code.py +++ b/Lib/code.py @@ -13,7 +13,6 @@ __all__ = ["InteractiveInterpreter", "InteractiveConsole", "interact", "compile_command"] - class InteractiveInterpreter: """Base class for InteractiveConsole. @@ -126,7 +125,7 @@ def showtraceback(self): """ try: typ, value, tb = sys.exc_info() - self._showtraceback(typ, value, tb.tb_next, '') + self._showtraceback(typ, value, tb.tb_next, "") finally: typ = value = tb = None @@ -140,7 +139,7 @@ def _showtraceback(self, typ, value, tb, source): and not value.text and value.lineno is not None and len(lines) >= value.lineno): value.text = lines[value.lineno - 1] - sys.last_exc = sys.last_value = value = value.with_traceback(tb) + sys.last_exc = sys.last_value = value if sys.excepthook is sys.__excepthook__: self._excepthook(typ, value, tb) else: @@ -220,12 +219,17 @@ def interact(self, banner=None, exitmsg=None): """ try: sys.ps1 + delete_ps1_after = False except AttributeError: sys.ps1 = ">>> " + delete_ps1_after = True try: - sys.ps2 + _ps2 = sys.ps2 + delete_ps2_after = False except AttributeError: sys.ps2 = "... " + delete_ps2_after = True + cprt = 'Type "help", "copyright", "credits" or "license" for more information.' if banner is None: self.write("Python %s on %s\n%s\n(%s)\n" % @@ -288,6 +292,12 @@ def interact(self, banner=None, exitmsg=None): if _quit is not None: builtins.quit = _quit + if delete_ps1_after: + del sys.ps1 + + if delete_ps2_after: + del sys.ps2 + if exitmsg is None: self.write('now exiting %s...\n' % self.__class__.__name__) elif exitmsg != '': @@ -366,7 +376,7 @@ def interact(banner=None, readfunc=None, local=None, exitmsg=None, local_exit=Fa console.raw_input = readfunc else: try: - import readline + import readline # noqa: F401 except ImportError: pass console.interact(banner, exitmsg) @@ -375,9 +385,9 @@ def interact(banner=None, readfunc=None, local=None, exitmsg=None, local_exit=Fa if __name__ == "__main__": import argparse - parser = argparse.ArgumentParser() + parser = argparse.ArgumentParser(color=True) parser.add_argument('-q', action='store_true', - help="don't print version and copyright messages") + help="don't print version and copyright messages") args = parser.parse_args() if args.q or sys.flags.quiet: banner = '' diff --git a/Lib/codecs.py b/Lib/codecs.py index e4f4e1b5c02..e4a8010aba9 100644 --- a/Lib/codecs.py +++ b/Lib/codecs.py @@ -884,7 +884,6 @@ def __reduce_ex__(self, proto): ### Shortcuts def open(filename, mode='r', encoding=None, errors='strict', buffering=-1): - """ Open an encoded file using the given mode and return a wrapped version providing transparent encoding/decoding. @@ -912,8 +911,11 @@ def open(filename, mode='r', encoding=None, errors='strict', buffering=-1): .encoding which allows querying the used encoding. This attribute is only available if an encoding was specified as parameter. - """ + import warnings + warnings.warn("codecs.open() is deprecated. Use open() instead.", + DeprecationWarning, stacklevel=2) + if encoding is not None and \ 'b' not in mode: # Force opening of the file in binary mode @@ -1109,24 +1111,15 @@ def make_encoding_map(decoding_map): ### error handlers -try: - strict_errors = lookup_error("strict") - ignore_errors = lookup_error("ignore") - replace_errors = lookup_error("replace") - xmlcharrefreplace_errors = lookup_error("xmlcharrefreplace") - backslashreplace_errors = lookup_error("backslashreplace") - namereplace_errors = lookup_error("namereplace") -except LookupError: - # In --disable-unicode builds, these error handler are missing - strict_errors = None - ignore_errors = None - replace_errors = None - xmlcharrefreplace_errors = None - backslashreplace_errors = None - namereplace_errors = None +strict_errors = lookup_error("strict") +ignore_errors = lookup_error("ignore") +replace_errors = lookup_error("replace") +xmlcharrefreplace_errors = lookup_error("xmlcharrefreplace") +backslashreplace_errors = lookup_error("backslashreplace") +namereplace_errors = lookup_error("namereplace") # Tell modulefinder that using codecs probably needs the encodings # package _false = 0 if _false: - import encodings + import encodings # noqa: F401 diff --git a/Lib/codeop.py b/Lib/codeop.py index adf000ba29f..8cac00442d9 100644 --- a/Lib/codeop.py +++ b/Lib/codeop.py @@ -47,7 +47,7 @@ PyCF_ONLY_AST = 0x400 PyCF_ALLOW_INCOMPLETE_INPUT = 0x4000 -def _maybe_compile(compiler, source, filename, symbol): +def _maybe_compile(compiler, source, filename, symbol, flags): # Check for source consisting of only blank lines and comments. for line in source.split("\n"): line = line.strip() @@ -61,10 +61,10 @@ def _maybe_compile(compiler, source, filename, symbol): with warnings.catch_warnings(): warnings.simplefilter("ignore", (SyntaxWarning, DeprecationWarning)) try: - compiler(source, filename, symbol) + compiler(source, filename, symbol, flags=flags) except SyntaxError: # Let other compile() errors propagate. try: - compiler(source + "\n", filename, symbol) + compiler(source + "\n", filename, symbol, flags=flags) return None except _IncompleteInputError as e: return None @@ -74,14 +74,13 @@ def _maybe_compile(compiler, source, filename, symbol): return compiler(source, filename, symbol, incomplete_input=False) -def _compile(source, filename, symbol, incomplete_input=True): - flags = 0 +def _compile(source, filename, symbol, incomplete_input=True, *, flags=0): if incomplete_input: flags |= PyCF_ALLOW_INCOMPLETE_INPUT flags |= PyCF_DONT_IMPLY_DEDENT return compile(source, filename, symbol, flags) -def compile_command(source, filename="", symbol="single"): +def compile_command(source, filename="", symbol="single", flags=0): r"""Compile a command and determine whether it is incomplete. Arguments: @@ -100,7 +99,7 @@ def compile_command(source, filename="", symbol="single"): syntax error (OverflowError and ValueError can be produced by malformed literals). """ - return _maybe_compile(_compile, source, filename, symbol) + return _maybe_compile(_compile, source, filename, symbol, flags) class Compile: """Instances of this class behave much like the built-in compile @@ -152,4 +151,4 @@ def __call__(self, source, filename="", symbol="single"): syntax error (OverflowError and ValueError can be produced by malformed literals). """ - return _maybe_compile(self.compiler, source, filename, symbol) + return _maybe_compile(self.compiler, source, filename, symbol, flags=self.compiler.flags) diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index db36248c4ff..3d3bbd7a39a 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -57,10 +57,7 @@ try: from _collections import defaultdict except ImportError: - # FIXME: try to implement defaultdict in collections.rs rather than in Python - # I (coolreader18) couldn't figure out some class stuff with __new__ and - # __init__ and __missing__ and subclassing built-in types from Rust, so I went - # with this instead. + # TODO: RUSTPYTHON - implement defaultdict in Rust from ._defaultdict import defaultdict heapq = None # Lazily imported diff --git a/Lib/compileall.py b/Lib/compileall.py index 47e2446356e..67fe370451e 100644 --- a/Lib/compileall.py +++ b/Lib/compileall.py @@ -317,7 +317,9 @@ def main(): import argparse parser = argparse.ArgumentParser( - description='Utilities to support installing Python libraries.') + description='Utilities to support installing Python libraries.', + color=True, + ) parser.add_argument('-l', action='store_const', const=0, default=None, dest='maxlevels', help="don't recurse into subdirectories") diff --git a/Lib/configparser.py b/Lib/configparser.py index 05b86acb919..d435a5c2fe0 100644 --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -154,14 +154,13 @@ import os import re import sys -import types __all__ = ("NoSectionError", "DuplicateOptionError", "DuplicateSectionError", "NoOptionError", "InterpolationError", "InterpolationDepthError", "InterpolationMissingOptionError", "InterpolationSyntaxError", "ParsingError", "MissingSectionHeaderError", - "MultilineContinuationError", - "ConfigParser", "RawConfigParser", + "MultilineContinuationError", "UnnamedSectionDisabledError", + "InvalidWriteError", "ConfigParser", "RawConfigParser", "Interpolation", "BasicInterpolation", "ExtendedInterpolation", "SectionProxy", "ConverterMapping", "DEFAULTSECT", "MAX_INTERPOLATION_DEPTH", "UNNAMED_SECTION") @@ -362,11 +361,27 @@ def __init__(self, filename, lineno, line): self.line = line self.args = (filename, lineno, line) + +class UnnamedSectionDisabledError(Error): + """Raised when an attempt to use UNNAMED_SECTION is made with the + feature disabled.""" + def __init__(self): + Error.__init__(self, "Support for UNNAMED_SECTION is disabled.") + + class _UnnamedSection: def __repr__(self): return "" +class InvalidWriteError(Error): + """Raised when attempting to write data that the parser would read back differently. + ex: writing a key which begins with the section header pattern would read back as a + new section """ + + def __init__(self, msg=''): + Error.__init__(self, msg) + UNNAMED_SECTION = _UnnamedSection() @@ -556,35 +571,36 @@ def __init__(self): class _Line(str): + __slots__ = 'clean', 'has_comments' def __new__(cls, val, *args, **kwargs): return super().__new__(cls, val) - def __init__(self, val, prefixes): - self.prefixes = prefixes - - @functools.cached_property - def clean(self): - return self._strip_full() and self._strip_inline() + def __init__(self, val, comments): + trimmed = val.strip() + self.clean = comments.strip(trimmed) + self.has_comments = trimmed != self.clean - @property - def has_comments(self): - return self.strip() != self.clean - def _strip_inline(self): - """ - Search for the earliest prefix at the beginning of the line or following a space. - """ - matcher = re.compile( - '|'.join(fr'(^|\s)({re.escape(prefix)})' for prefix in self.prefixes.inline) - # match nothing if no prefixes - or '(?!)' +class _CommentSpec: + def __init__(self, full_prefixes, inline_prefixes): + full_patterns = ( + # prefix at the beginning of a line + fr'^({re.escape(prefix)}).*' + for prefix in full_prefixes ) - match = matcher.search(self) - return self[:match.start() if match else None].strip() + inline_patterns = ( + # prefix at the beginning of the line or following a space + fr'(^|\s)({re.escape(prefix)}.*)' + for prefix in inline_prefixes + ) + self.pattern = re.compile('|'.join(itertools.chain(full_patterns, inline_patterns))) + + def strip(self, text): + return self.pattern.sub('', text).rstrip() - def _strip_full(self): - return '' if any(map(self.strip().startswith, self.prefixes.full)) else True + def wrap(self, text): + return _Line(text, self) class RawConfigParser(MutableMapping): @@ -653,10 +669,7 @@ def __init__(self, defaults=None, dict_type=_default_dict, else: self._optcre = re.compile(self._OPT_TMPL.format(delim=d), re.VERBOSE) - self._prefixes = types.SimpleNamespace( - full=tuple(comment_prefixes or ()), - inline=tuple(inline_comment_prefixes or ()), - ) + self._comments = _CommentSpec(comment_prefixes or (), inline_comment_prefixes or ()) self._strict = strict self._allow_no_value = allow_no_value self._empty_lines_in_values = empty_lines_in_values @@ -694,6 +707,10 @@ def add_section(self, section): if section == self.default_section: raise ValueError('Invalid section name: %r' % section) + if section is UNNAMED_SECTION: + if not self._allow_unnamed_section: + raise UnnamedSectionDisabledError + if section in self._sections: raise DuplicateSectionError(section) self._sections[section] = self._dict() @@ -777,7 +794,8 @@ def read_dict(self, dictionary, source=''): """ elements_added = set() for section, keys in dictionary.items(): - section = str(section) + if section is not UNNAMED_SECTION: + section = str(section) try: self.add_section(section) except (DuplicateSectionError, ValueError): @@ -949,7 +967,7 @@ def write(self, fp, space_around_delimiters=True): if self._defaults: self._write_section(fp, self.default_section, self._defaults.items(), d) - if UNNAMED_SECTION in self._sections: + if UNNAMED_SECTION in self._sections and self._sections[UNNAMED_SECTION]: self._write_section(fp, UNNAMED_SECTION, self._sections[UNNAMED_SECTION].items(), d, unnamed=True) for section in self._sections: @@ -959,10 +977,11 @@ def write(self, fp, space_around_delimiters=True): self._sections[section].items(), d) def _write_section(self, fp, section_name, section_items, delimiter, unnamed=False): - """Write a single section to the specified `fp'.""" + """Write a single section to the specified 'fp'.""" if not unnamed: fp.write("[{}]\n".format(section_name)) for key, value in section_items: + self._validate_key_contents(key) value = self._interpolation.before_write(self, section_name, key, value) if value is not None or not self._allow_no_value: @@ -1047,7 +1066,6 @@ def _read(self, fp, fpname): in an otherwise empty line or may be entered in lines holding values or section names. Please note that comments get stripped off when reading configuration files. """ - try: ParsingError._raise_all(self._read_inner(fp, fpname)) finally: @@ -1056,8 +1074,7 @@ def _read(self, fp, fpname): def _read_inner(self, fp, fpname): st = _ReadState() - Line = functools.partial(_Line, prefixes=self._prefixes) - for st.lineno, line in enumerate(map(Line, fp), start=1): + for st.lineno, line in enumerate(map(self._comments.wrap, fp), start=1): if not line.clean: if self._empty_lines_in_values: # add empty line to the value, but only if there was no @@ -1200,21 +1217,32 @@ def _convert_to_boolean(self, value): raise ValueError('Not a boolean: %s' % value) return self.BOOLEAN_STATES[value.lower()] + def _validate_key_contents(self, key): + """Raises an InvalidWriteError for any keys containing + delimiters or that begins with the section header pattern""" + if re.match(self.SECTCRE, key): + raise InvalidWriteError( + f"Cannot write key {key}; begins with section pattern") + for delim in self._delimiters: + if delim in key: + raise InvalidWriteError( + f"Cannot write key {key}; contains delimiter {delim}") + def _validate_value_types(self, *, section="", option="", value=""): - """Raises a TypeError for non-string values. + """Raises a TypeError for illegal non-string values. - The only legal non-string value if we allow valueless - options is None, so we need to check if the value is a - string if: - - we do not allow valueless options, or - - we allow valueless options but the value is not None + Legal non-string values are UNNAMED_SECTION and falsey values if + they are allowed. For compatibility reasons this method is not used in classic set() for RawConfigParsers. It is invoked in every case for mapping protocol access and in ConfigParser.set(). """ - if not isinstance(section, str): - raise TypeError("section names must be strings") + if section is UNNAMED_SECTION: + if not self._allow_unnamed_section: + raise UnnamedSectionDisabledError + elif not isinstance(section, str): + raise TypeError("section names must be strings or UNNAMED_SECTION") if not isinstance(option, str): raise TypeError("option keys must be strings") if not self._allow_no_value or value: diff --git a/Lib/copy.py b/Lib/copy.py index 2a4606246aa..c64fc076179 100644 --- a/Lib/copy.py +++ b/Lib/copy.py @@ -67,13 +67,15 @@ def copy(x): cls = type(x) - copier = _copy_dispatch.get(cls) - if copier: - return copier(x) + if cls in _copy_atomic_types: + return x + if cls in _copy_builtin_containers: + return cls.copy(x) + if issubclass(cls, type): # treat it as a regular class: - return _copy_immutable(x) + return x copier = getattr(cls, "__copy__", None) if copier is not None: @@ -98,23 +100,12 @@ def copy(x): return _reconstruct(x, None, *rv) -_copy_dispatch = d = {} - -def _copy_immutable(x): - return x -for t in (types.NoneType, int, float, bool, complex, str, tuple, +_copy_atomic_types = {types.NoneType, int, float, bool, complex, str, tuple, bytes, frozenset, type, range, slice, property, types.BuiltinFunctionType, types.EllipsisType, types.NotImplementedType, types.FunctionType, types.CodeType, - weakref.ref): - d[t] = _copy_immutable - -d[list] = list.copy -d[dict] = dict.copy -d[set] = set.copy -d[bytearray] = bytearray.copy - -del d, t + weakref.ref, super} +_copy_builtin_containers = {list, dict, set, bytearray} def deepcopy(x, memo=None, _nil=[]): """Deep copy operation on arbitrary Python objects. @@ -122,6 +113,11 @@ def deepcopy(x, memo=None, _nil=[]): See the module's __doc__ string for more info. """ + cls = type(x) + + if cls in _atomic_types: + return x + d = id(x) if memo is None: memo = {} @@ -130,14 +126,12 @@ def deepcopy(x, memo=None, _nil=[]): if y is not _nil: return y - cls = type(x) - copier = _deepcopy_dispatch.get(cls) if copier is not None: y = copier(x, memo) else: if issubclass(cls, type): - y = _deepcopy_atomic(x, memo) + y = x # atomic copy else: copier = getattr(x, "__deepcopy__", None) if copier is not None: @@ -168,26 +162,12 @@ def deepcopy(x, memo=None, _nil=[]): _keep_alive(x, memo) # Make sure x lives at least as long as d return y +_atomic_types = {types.NoneType, types.EllipsisType, types.NotImplementedType, + int, float, bool, complex, bytes, str, types.CodeType, type, range, + types.BuiltinFunctionType, types.FunctionType, weakref.ref, property} + _deepcopy_dispatch = d = {} -def _deepcopy_atomic(x, memo): - return x -d[types.NoneType] = _deepcopy_atomic -d[types.EllipsisType] = _deepcopy_atomic -d[types.NotImplementedType] = _deepcopy_atomic -d[int] = _deepcopy_atomic -d[float] = _deepcopy_atomic -d[bool] = _deepcopy_atomic -d[complex] = _deepcopy_atomic -d[bytes] = _deepcopy_atomic -d[str] = _deepcopy_atomic -d[types.CodeType] = _deepcopy_atomic -d[type] = _deepcopy_atomic -d[range] = _deepcopy_atomic -d[types.BuiltinFunctionType] = _deepcopy_atomic -d[types.FunctionType] = _deepcopy_atomic -d[weakref.ref] = _deepcopy_atomic -d[property] = _deepcopy_atomic def _deepcopy_list(x, memo, deepcopy=deepcopy): y = [] diff --git a/Lib/csv.py b/Lib/csv.py index cd202659873..0a627ba7a51 100644 --- a/Lib/csv.py +++ b/Lib/csv.py @@ -63,7 +63,6 @@ class excel: written as two quotes """ -import re import types from _csv import Error, writer, reader, register_dialect, \ unregister_dialect, get_dialect, list_dialects, \ @@ -281,6 +280,7 @@ def _guess_quote_and_delimiter(self, data, delimiters): If there is no quotechar the delimiter can't be determined this way. """ + import re matches = [] for restr in (r'(?P[^\w\n"\'])(?P ?)(?P["\']).*?(?P=quote)(?P=delim)', # ,".*?", diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py index 80651dc64ce..04ec0270148 100644 --- a/Lib/ctypes/__init__.py +++ b/Lib/ctypes/__init__.py @@ -14,6 +14,7 @@ from _ctypes import RTLD_LOCAL, RTLD_GLOBAL from _ctypes import ArgumentError from _ctypes import SIZEOF_TIME_T +from _ctypes import CField from struct import calcsize as _calcsize @@ -21,7 +22,7 @@ raise Exception("Version number mismatch", __version__, _ctypes_version) if _os.name == "nt": - from _ctypes import FormatError + from _ctypes import COMError, CopyComPointer, FormatError DEFAULT_MODE = RTLD_LOCAL if _os.name == "posix" and _sys.platform == "darwin": @@ -163,6 +164,7 @@ def __repr__(self): return super().__repr__() except ValueError: return "%s()" % type(self).__name__ + __class_getitem__ = classmethod(_types.GenericAlias) _check_size(py_object, "P") class c_short(_SimpleCData): @@ -207,6 +209,18 @@ class c_longdouble(_SimpleCData): if sizeof(c_longdouble) == sizeof(c_double): c_longdouble = c_double +try: + class c_double_complex(_SimpleCData): + _type_ = "D" + _check_size(c_double_complex) + class c_float_complex(_SimpleCData): + _type_ = "F" + _check_size(c_float_complex) + class c_longdouble_complex(_SimpleCData): + _type_ = "G" +except AttributeError: + pass + if _calcsize("l") == _calcsize("q"): # if long and long long have the same size, make c_longlong an alias for c_long c_longlong = c_long @@ -254,7 +268,72 @@ class c_void_p(_SimpleCData): class c_bool(_SimpleCData): _type_ = "?" -from _ctypes import POINTER, pointer, _pointer_type_cache +def POINTER(cls): + """Create and return a new ctypes pointer type. + + Pointer types are cached and reused internally, + so calling this function repeatedly is cheap. + """ + if cls is None: + return c_void_p + try: + return cls.__pointer_type__ + except AttributeError: + pass + if isinstance(cls, str): + # handle old-style incomplete types (see test_ctypes.test_incomplete) + import warnings + warnings._deprecated("ctypes.POINTER with string", remove=(3, 19)) + try: + return _pointer_type_cache_fallback[cls] + except KeyError: + result = type(f'LP_{cls}', (_Pointer,), {}) + _pointer_type_cache_fallback[cls] = result + return result + + # create pointer type and set __pointer_type__ for cls + return type(f'LP_{cls.__name__}', (_Pointer,), {'_type_': cls}) + +def pointer(obj): + """Create a new pointer instance, pointing to 'obj'. + + The returned object is of the type POINTER(type(obj)). Note that if you + just want to pass a pointer to an object to a foreign function call, you + should use byref(obj) which is much faster. + """ + typ = POINTER(type(obj)) + return typ(obj) + +class _PointerTypeCache: + def __setitem__(self, cls, pointer_type): + import warnings + warnings._deprecated("ctypes._pointer_type_cache", remove=(3, 19)) + try: + cls.__pointer_type__ = pointer_type + except AttributeError: + _pointer_type_cache_fallback[cls] = pointer_type + + def __getitem__(self, cls): + import warnings + warnings._deprecated("ctypes._pointer_type_cache", remove=(3, 19)) + try: + return cls.__pointer_type__ + except AttributeError: + return _pointer_type_cache_fallback[cls] + + def get(self, cls, default=None): + import warnings + warnings._deprecated("ctypes._pointer_type_cache", remove=(3, 19)) + try: + return cls.__pointer_type__ + except AttributeError: + return _pointer_type_cache_fallback.get(cls, default) + + def __contains__(self, cls): + return hasattr(cls, '__pointer_type__') + +_pointer_type_cache_fallback = {} +_pointer_type_cache = _PointerTypeCache() class c_wchar_p(_SimpleCData): _type_ = "Z" @@ -265,7 +344,7 @@ class c_wchar(_SimpleCData): _type_ = "u" def _reset_cache(): - _pointer_type_cache.clear() + _pointer_type_cache_fallback.clear() _c_functype_cache.clear() if _os.name == "nt": _win_functype_cache.clear() @@ -273,7 +352,6 @@ def _reset_cache(): POINTER(c_wchar).from_param = c_wchar_p.from_param # _SimpleCData.c_char_p_from_param POINTER(c_char).from_param = c_char_p.from_param - _pointer_type_cache[None] = c_void_p def create_unicode_buffer(init, size=None): """create_unicode_buffer(aString) -> character array @@ -307,13 +385,7 @@ def create_unicode_buffer(init, size=None): def SetPointerType(pointer, cls): import warnings warnings._deprecated("ctypes.SetPointerType", remove=(3, 15)) - if _pointer_type_cache.get(cls, None) is not None: - raise RuntimeError("This type already exists in the cache") - if id(pointer) not in _pointer_type_cache: - raise RuntimeError("What's this???") pointer.set_type(cls) - _pointer_type_cache[cls] = pointer - del _pointer_type_cache[id(pointer)] def ARRAY(typ, len): return typ * len @@ -520,6 +592,7 @@ def WinError(code=None, descr=None): # functions from _ctypes import _memmove_addr, _memset_addr, _string_at_addr, _cast_addr +from _ctypes import _memoryview_at_addr ## void *memmove(void *, const void *, size_t); memmove = CFUNCTYPE(c_void_p, c_void_p, c_void_p, c_size_t)(_memmove_addr) @@ -545,6 +618,14 @@ def string_at(ptr, size=-1): Return the byte string at void *ptr.""" return _string_at(ptr, size) +_memoryview_at = PYFUNCTYPE( + py_object, c_void_p, c_ssize_t, c_int)(_memoryview_at_addr) +def memoryview_at(ptr, size, readonly=False): + """memoryview_at(ptr, size[, readonly]) -> memoryview + + Return a memoryview representing the memory at void *ptr.""" + return _memoryview_at(ptr, size, bool(readonly)) + try: from _ctypes import _wstring_at_addr except ImportError: diff --git a/Lib/ctypes/_layout.py b/Lib/ctypes/_layout.py new file mode 100644 index 00000000000..2048ccb6a1c --- /dev/null +++ b/Lib/ctypes/_layout.py @@ -0,0 +1,330 @@ +"""Python implementation of computing the layout of a struct/union + +This code is internal and tightly coupled to the C part. The interface +may change at any time. +""" + +import sys +import warnings + +from _ctypes import CField, buffer_info +import ctypes + +def round_down(n, multiple): + assert n >= 0 + assert multiple > 0 + return (n // multiple) * multiple + +def round_up(n, multiple): + assert n >= 0 + assert multiple > 0 + return ((n + multiple - 1) // multiple) * multiple + +_INT_MAX = (1 << (ctypes.sizeof(ctypes.c_int) * 8) - 1) - 1 + + +class StructUnionLayout: + def __init__(self, fields, size, align, format_spec): + # sequence of CField objects + self.fields = fields + + # total size of the aggregate (rounded up to alignment) + self.size = size + + # total alignment requirement of the aggregate + self.align = align + + # buffer format specification (as a string, UTF-8 but bes + # kept ASCII-only) + self.format_spec = format_spec + + +def get_layout(cls, input_fields, is_struct, base): + """Return a StructUnionLayout for the given class. + + Called by PyCStructUnionType_update_stginfo when _fields_ is assigned + to a class. + """ + # Currently there are two modes, selectable using the '_layout_' attribute: + # + # 'gcc-sysv' mode places fields one after another, bit by bit. + # But "each bit field must fit within a single object of its specified + # type" (GCC manual, section 15.8 "Bit Field Packing"). When it doesn't, + # we insert a few bits of padding to avoid that. + # + # 'ms' mode works similar except for bitfield packing. Adjacent + # bit-fields are packed into the same 1-, 2-, or 4-byte allocation unit + # if the integral types are the same size and if the next bit-field fits + # into the current allocation unit without crossing the boundary imposed + # by the common alignment requirements of the bit-fields. + # + # See https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html#index-mms-bitfields + # for details. + + # We do not support zero length bitfields (we use bitsize != 0 + # elsewhere to indicate a bitfield). Here, non-bitfields have bit_size + # set to size*8. + + # For clarity, variables that count bits have `bit` in their names. + + pack = getattr(cls, '_pack_', None) + + layout = getattr(cls, '_layout_', None) + if layout is None: + if sys.platform == 'win32': + gcc_layout = False + elif pack: + if is_struct: + base_type_name = 'Structure' + else: + base_type_name = 'Union' + warnings._deprecated( + '_pack_ without _layout_', + f"Due to '_pack_', the '{cls.__name__}' {base_type_name} will " + + "use memory layout compatible with MSVC (Windows). " + + "If this is intended, set _layout_ to 'ms'. " + + "The implicit default is deprecated and slated to become " + + "an error in Python {remove}.", + remove=(3, 19), + ) + gcc_layout = False + else: + gcc_layout = True + elif layout == 'ms': + gcc_layout = False + elif layout == 'gcc-sysv': + gcc_layout = True + else: + raise ValueError(f'unknown _layout_: {layout!r}') + + align = getattr(cls, '_align_', 1) + if align < 0: + raise ValueError('_align_ must be a non-negative integer') + elif align == 0: + # Setting `_align_ = 0` amounts to using the default alignment + align = 1 + + if base: + align = max(ctypes.alignment(base), align) + + swapped_bytes = hasattr(cls, '_swappedbytes_') + if swapped_bytes: + big_endian = sys.byteorder == 'little' + else: + big_endian = sys.byteorder == 'big' + + if pack is not None: + try: + pack = int(pack) + except (TypeError, ValueError): + raise ValueError("_pack_ must be an integer") + if pack < 0: + raise ValueError("_pack_ must be a non-negative integer") + if pack > _INT_MAX: + raise ValueError("_pack_ too big") + if gcc_layout: + raise ValueError('_pack_ is not compatible with gcc-sysv layout') + + result_fields = [] + + if is_struct: + format_spec_parts = ["T{"] + else: + format_spec_parts = ["B"] + + last_field_bit_size = 0 # used in MS layout only + + # `8 * next_byte_offset + next_bit_offset` points to where the + # next field would start. + next_bit_offset = 0 + next_byte_offset = 0 + + # size if this was a struct (sum of field sizes, plus padding) + struct_size = 0 + # max of field sizes; only meaningful for unions + union_size = 0 + + if base: + struct_size = ctypes.sizeof(base) + if gcc_layout: + next_bit_offset = struct_size * 8 + else: + next_byte_offset = struct_size + + last_size = struct_size + for i, field in enumerate(input_fields): + if not is_struct: + # Unions start fresh each time + last_field_bit_size = 0 + next_bit_offset = 0 + next_byte_offset = 0 + + # Unpack the field + field = tuple(field) + try: + name, ctype = field + except (ValueError, TypeError): + try: + name, ctype, bit_size = field + except (ValueError, TypeError) as exc: + raise ValueError( + '_fields_ must be a sequence of (name, C type) pairs ' + + 'or (name, C type, bit size) triples') from exc + is_bitfield = True + if bit_size <= 0: + raise ValueError( + f'number of bits invalid for bit field {name!r}') + type_size = ctypes.sizeof(ctype) + if bit_size > type_size * 8: + raise ValueError( + f'number of bits invalid for bit field {name!r}') + else: + is_bitfield = False + type_size = ctypes.sizeof(ctype) + bit_size = type_size * 8 + + type_bit_size = type_size * 8 + type_align = ctypes.alignment(ctype) or 1 + type_bit_align = type_align * 8 + + if gcc_layout: + # We don't use next_byte_offset here + assert pack is None + assert next_byte_offset == 0 + + # Determine whether the bit field, if placed at the next + # free bit, fits within a single object of its specified type. + # That is: determine a "slot", sized & aligned for the + # specified type, which contains the bitfield's beginning: + slot_start_bit = round_down(next_bit_offset, type_bit_align) + slot_end_bit = slot_start_bit + type_bit_size + # And see if it also contains the bitfield's last bit: + field_end_bit = next_bit_offset + bit_size + if field_end_bit > slot_end_bit: + # It doesn't: add padding (bump up to the next + # alignment boundary) + next_bit_offset = round_up(next_bit_offset, type_bit_align) + + offset = round_down(next_bit_offset, type_bit_align) // 8 + if is_bitfield: + bit_offset = next_bit_offset - 8 * offset + assert bit_offset <= type_bit_size + else: + assert offset == next_bit_offset / 8 + + next_bit_offset += bit_size + struct_size = round_up(next_bit_offset, 8) // 8 + else: + if pack: + type_align = min(pack, type_align) + + # next_byte_offset points to end of current bitfield. + # next_bit_offset is generally non-positive, + # and 8 * next_byte_offset + next_bit_offset points just behind + # the end of the last field we placed. + if ( + (0 < next_bit_offset + bit_size) + or (type_bit_size != last_field_bit_size) + ): + # Close the previous bitfield (if any) + # and start a new bitfield + next_byte_offset = round_up(next_byte_offset, type_align) + + next_byte_offset += type_size + + last_field_bit_size = type_bit_size + # Reminder: 8 * (next_byte_offset) + next_bit_offset + # points to where we would start a new field, namely + # just behind where we placed the last field plus an + # allowance for alignment. + next_bit_offset = -last_field_bit_size + + assert type_bit_size == last_field_bit_size + + offset = next_byte_offset - last_field_bit_size // 8 + if is_bitfield: + assert 0 <= (last_field_bit_size + next_bit_offset) + bit_offset = last_field_bit_size + next_bit_offset + if type_bit_size: + assert (last_field_bit_size + next_bit_offset) < type_bit_size + + next_bit_offset += bit_size + struct_size = next_byte_offset + + if is_bitfield and big_endian: + # On big-endian architectures, bit fields are also laid out + # starting with the big end. + bit_offset = type_bit_size - bit_size - bit_offset + + # Add the format spec parts + if is_struct: + padding = offset - last_size + format_spec_parts.append(padding_spec(padding)) + + fieldfmt, bf_ndim, bf_shape = buffer_info(ctype) + + if bf_shape: + format_spec_parts.extend(( + "(", + ','.join(str(n) for n in bf_shape), + ")", + )) + + if fieldfmt is None: + fieldfmt = "B" + if isinstance(name, bytes): + # a bytes name would be rejected later, but we check early + # to avoid a BytesWarning with `python -bb` + raise TypeError( + f"field {name!r}: name must be a string, not bytes") + format_spec_parts.append(f"{fieldfmt}:{name}:") + + result_fields.append(CField( + name=name, + type=ctype, + byte_size=type_size, + byte_offset=offset, + bit_size=bit_size if is_bitfield else None, + bit_offset=bit_offset if is_bitfield else None, + index=i, + + # Do not use CField outside ctypes, yet. + # The constructor is internal API and may change without warning. + _internal_use=True, + )) + if is_bitfield and not gcc_layout: + assert type_bit_size > 0 + + align = max(align, type_align) + last_size = struct_size + if not is_struct: + union_size = max(struct_size, union_size) + + if is_struct: + total_size = struct_size + else: + total_size = union_size + + # Adjust the size according to the alignment requirements + aligned_size = round_up(total_size, align) + + # Finish up the format spec + if is_struct: + padding = aligned_size - total_size + format_spec_parts.append(padding_spec(padding)) + format_spec_parts.append("}") + + return StructUnionLayout( + fields=result_fields, + size=aligned_size, + align=align, + format_spec="".join(format_spec_parts), + ) + + +def padding_spec(padding): + if padding <= 0: + return "" + if padding == 1: + return "x" + return f"{padding}x" diff --git a/Lib/ctypes/util.py b/Lib/ctypes/util.py index 117bf06cb01..378f12167c6 100644 --- a/Lib/ctypes/util.py +++ b/Lib/ctypes/util.py @@ -67,6 +67,65 @@ def find_library(name): return fname return None + # Listing loaded DLLs on Windows relies on the following APIs: + # https://learn.microsoft.com/windows/win32/api/psapi/nf-psapi-enumprocessmodules + # https://learn.microsoft.com/windows/win32/api/libloaderapi/nf-libloaderapi-getmodulefilenamew + import ctypes + from ctypes import wintypes + + _kernel32 = ctypes.WinDLL('kernel32', use_last_error=True) + _get_current_process = _kernel32["GetCurrentProcess"] + _get_current_process.restype = wintypes.HANDLE + + _k32_get_module_file_name = _kernel32["GetModuleFileNameW"] + _k32_get_module_file_name.restype = wintypes.DWORD + _k32_get_module_file_name.argtypes = ( + wintypes.HMODULE, + wintypes.LPWSTR, + wintypes.DWORD, + ) + + _psapi = ctypes.WinDLL('psapi', use_last_error=True) + _enum_process_modules = _psapi["EnumProcessModules"] + _enum_process_modules.restype = wintypes.BOOL + _enum_process_modules.argtypes = ( + wintypes.HANDLE, + ctypes.POINTER(wintypes.HMODULE), + wintypes.DWORD, + wintypes.LPDWORD, + ) + + def _get_module_filename(module: wintypes.HMODULE): + name = (wintypes.WCHAR * 32767)() # UNICODE_STRING_MAX_CHARS + if _k32_get_module_file_name(module, name, len(name)): + return name.value + return None + + + def _get_module_handles(): + process = _get_current_process() + space_needed = wintypes.DWORD() + n = 1024 + while True: + modules = (wintypes.HMODULE * n)() + if not _enum_process_modules(process, + modules, + ctypes.sizeof(modules), + ctypes.byref(space_needed)): + err = ctypes.get_last_error() + msg = ctypes.FormatError(err).strip() + raise ctypes.WinError(err, f"EnumProcessModules failed: {msg}") + n = space_needed.value // ctypes.sizeof(wintypes.HMODULE) + if n <= len(modules): + return modules[:n] + + def dllist(): + """Return a list of loaded shared libraries in the current process.""" + modules = _get_module_handles() + libraries = [name for h in modules + if (name := _get_module_filename(h)) is not None] + return libraries + elif os.name == "posix" and sys.platform in {"darwin", "ios", "tvos", "watchos"}: from ctypes.macholib.dyld import dyld_find as _dyld_find def find_library(name): @@ -80,6 +139,22 @@ def find_library(name): continue return None + # Listing loaded libraries on Apple systems relies on the following API: + # https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/dyld.3.html + import ctypes + + _libc = ctypes.CDLL(find_library("c")) + _dyld_get_image_name = _libc["_dyld_get_image_name"] + _dyld_get_image_name.restype = ctypes.c_char_p + + def dllist(): + """Return a list of loaded shared libraries in the current process.""" + num_images = _libc._dyld_image_count() + libraries = [os.fsdecode(name) for i in range(num_images) + if (name := _dyld_get_image_name(i)) is not None] + + return libraries + elif sys.platform.startswith("aix"): # AIX has two styles of storing shared libraries # GNU auto_tools refer to these as svr4 and aix @@ -98,6 +173,25 @@ def find_library(name): fname = f"{directory}/lib{name}.so" return fname if os.path.isfile(fname) else None +elif sys.platform == "emscripten": + def _is_wasm(filename): + # Return True if the given file is an WASM module + wasm_header = b"\x00asm" + with open(filename, 'br') as thefile: + return thefile.read(4) == wasm_header + + def find_library(name): + candidates = [f"lib{name}.so", f"lib{name}.wasm"] + paths = os.environ.get("LD_LIBRARY_PATH", "") + for libdir in paths.split(":"): + for name in candidates: + libfile = os.path.join(libdir, name) + + if os.path.isfile(libfile) and _is_wasm(libfile): + return libfile + + return None + elif os.name == "posix": # Andreas Degert's find functions, using gcc, /sbin/ldconfig, objdump import re, tempfile @@ -341,6 +435,55 @@ def find_library(name): return _findSoname_ldconfig(name) or \ _get_soname(_findLib_gcc(name)) or _get_soname(_findLib_ld(name)) + +# Listing loaded libraries on other systems will try to use +# functions common to Linux and a few other Unix-like systems. +# See the following for several platforms' documentation of the same API: +# https://man7.org/linux/man-pages/man3/dl_iterate_phdr.3.html +# https://man.freebsd.org/cgi/man.cgi?query=dl_iterate_phdr +# https://man.openbsd.org/dl_iterate_phdr +# https://docs.oracle.com/cd/E88353_01/html/E37843/dl-iterate-phdr-3c.html +if (os.name == "posix" and + sys.platform not in {"darwin", "ios", "tvos", "watchos"}): + import ctypes + if hasattr((_libc := ctypes.CDLL(None)), "dl_iterate_phdr"): + + class _dl_phdr_info(ctypes.Structure): + _fields_ = [ + ("dlpi_addr", ctypes.c_void_p), + ("dlpi_name", ctypes.c_char_p), + ("dlpi_phdr", ctypes.c_void_p), + ("dlpi_phnum", ctypes.c_ushort), + ] + + _dl_phdr_callback = ctypes.CFUNCTYPE( + ctypes.c_int, + ctypes.POINTER(_dl_phdr_info), + ctypes.c_size_t, + ctypes.POINTER(ctypes.py_object), + ) + + @_dl_phdr_callback + def _info_callback(info, _size, data): + libraries = data.contents.value + name = os.fsdecode(info.contents.dlpi_name) + libraries.append(name) + return 0 + + _dl_iterate_phdr = _libc["dl_iterate_phdr"] + _dl_iterate_phdr.argtypes = [ + _dl_phdr_callback, + ctypes.POINTER(ctypes.py_object), + ] + _dl_iterate_phdr.restype = ctypes.c_int + + def dllist(): + """Return a list of loaded shared libraries in the current process.""" + libraries = [] + _dl_iterate_phdr(_info_callback, + ctypes.byref(ctypes.py_object(libraries))) + return libraries + ################################################################ # test code @@ -384,5 +527,12 @@ def test(): print(cdll.LoadLibrary("libcrypt.so")) print(find_library("crypt")) + try: + dllist + except NameError: + print('dllist() not available') + else: + print(dllist()) + if __name__ == "__main__": test() diff --git a/Lib/ctypes/wintypes.py b/Lib/ctypes/wintypes.py index 9c4e721438a..4beba0d1951 100644 --- a/Lib/ctypes/wintypes.py +++ b/Lib/ctypes/wintypes.py @@ -63,10 +63,16 @@ def __repr__(self): HBITMAP = HANDLE HBRUSH = HANDLE HCOLORSPACE = HANDLE +HCONV = HANDLE +HCONVLIST = HANDLE +HCURSOR = HANDLE HDC = HANDLE +HDDEDATA = HANDLE HDESK = HANDLE +HDROP = HANDLE HDWP = HANDLE HENHMETAFILE = HANDLE +HFILE = INT HFONT = HANDLE HGDIOBJ = HANDLE HGLOBAL = HANDLE @@ -82,9 +88,11 @@ def __repr__(self): HMONITOR = HANDLE HPALETTE = HANDLE HPEN = HANDLE +HRESULT = LONG HRGN = HANDLE HRSRC = HANDLE HSTR = HANDLE +HSZ = HANDLE HTASK = HANDLE HWINSTA = HANDLE HWND = HANDLE diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 7883ce78e57..c8dbb247745 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -5,6 +5,7 @@ import inspect import keyword import itertools +import annotationlib import abc from reprlib import recursive_repr @@ -243,6 +244,10 @@ def __repr__(self): property, }) +# Any marker is used in `make_dataclass` to mark unannotated fields as `Any` +# without importing `typing` module. +_ANY_MARKER = object() + class InitVar: __slots__ = ('type', ) @@ -282,11 +287,12 @@ class Field: 'compare', 'metadata', 'kw_only', + 'doc', '_field_type', # Private: not to be used by user code. ) def __init__(self, default, default_factory, init, repr, hash, compare, - metadata, kw_only): + metadata, kw_only, doc): self.name = None self.type = None self.default = default @@ -299,6 +305,7 @@ def __init__(self, default, default_factory, init, repr, hash, compare, if metadata is None else types.MappingProxyType(metadata)) self.kw_only = kw_only + self.doc = doc self._field_type = None @recursive_repr() @@ -314,6 +321,7 @@ def __repr__(self): f'compare={self.compare!r},' f'metadata={self.metadata!r},' f'kw_only={self.kw_only!r},' + f'doc={self.doc!r},' f'_field_type={self._field_type}' ')') @@ -381,7 +389,7 @@ def __repr__(self): # so that a type checker can be told (via overloads) that this is a # function whose type depends on its parameters. def field(*, default=MISSING, default_factory=MISSING, init=True, repr=True, - hash=None, compare=True, metadata=None, kw_only=MISSING): + hash=None, compare=True, metadata=None, kw_only=MISSING, doc=None): """Return an object to identify dataclass fields. default is the default value of the field. default_factory is a @@ -393,7 +401,7 @@ def field(*, default=MISSING, default_factory=MISSING, init=True, repr=True, comparison functions. metadata, if specified, must be a mapping which is stored but not otherwise examined by dataclass. If kw_only is true, the field will become a keyword-only parameter to - __init__(). + __init__(). doc is an optional docstring for this field. It is an error to specify both default and default_factory. """ @@ -401,7 +409,7 @@ def field(*, default=MISSING, default_factory=MISSING, init=True, repr=True, if default is not MISSING and default_factory is not MISSING: raise ValueError('cannot specify both default and default_factory') return Field(default, default_factory, init, repr, hash, compare, - metadata, kw_only) + metadata, kw_only, doc) def _fields_in_init_order(fields): @@ -433,9 +441,11 @@ def __init__(self, globals): self.locals = {} self.overwrite_errors = {} self.unconditional_adds = {} + self.method_annotations = {} def add_fn(self, name, args, body, *, locals=None, return_type=MISSING, - overwrite_error=False, unconditional_add=False, decorator=None): + overwrite_error=False, unconditional_add=False, decorator=None, + annotation_fields=None): if locals is not None: self.locals.update(locals) @@ -456,16 +466,14 @@ def add_fn(self, name, args, body, *, locals=None, return_type=MISSING, self.names.append(name) - if return_type is not MISSING: - self.locals[f'__dataclass_{name}_return_type__'] = return_type - return_annotation = f'->__dataclass_{name}_return_type__' - else: - return_annotation = '' + if annotation_fields is not None: + self.method_annotations[name] = (annotation_fields, return_type) + args = ','.join(args) body = '\n'.join(body) # Compute the text of the entire function, add it to the text we're generating. - self.src.append(f'{f' {decorator}\n' if decorator else ''} def {name}({args}){return_annotation}:\n{body}') + self.src.append(f'{f' {decorator}\n' if decorator else ''} def {name}({args}):\n{body}') def add_fns_to_class(self, cls): # The source to all of the functions we're generating. @@ -501,6 +509,15 @@ def add_fns_to_class(self, cls): # Now that we've generated the functions, assign them into cls. for name, fn in zip(self.names, fns): fn.__qualname__ = f"{cls.__qualname__}.{fn.__name__}" + + try: + annotation_fields, return_type = self.method_annotations[name] + except KeyError: + pass + else: + annotate_fn = _make_annotate_function(cls, name, annotation_fields, return_type) + fn.__annotate__ = annotate_fn + if self.unconditional_adds.get(name, False): setattr(cls, name, fn) else: @@ -516,6 +533,49 @@ def add_fns_to_class(self, cls): raise TypeError(error_msg) +def _make_annotate_function(__class__, method_name, annotation_fields, return_type): + # Create an __annotate__ function for a dataclass + # Try to return annotations in the same format as they would be + # from a regular __init__ function + + def __annotate__(format, /): + Format = annotationlib.Format + match format: + case Format.VALUE | Format.FORWARDREF | Format.STRING: + cls_annotations = {} + for base in reversed(__class__.__mro__): + cls_annotations.update( + annotationlib.get_annotations(base, format=format) + ) + + new_annotations = {} + for k in annotation_fields: + # gh-142214: The annotation may be missing in unusual dynamic cases. + # If so, just skip it. + try: + new_annotations[k] = cls_annotations[k] + except KeyError: + pass + + if return_type is not MISSING: + if format == Format.STRING: + new_annotations["return"] = annotationlib.type_repr(return_type) + else: + new_annotations["return"] = return_type + + return new_annotations + + case _: + raise NotImplementedError(format) + + # This is a flag for _add_slots to know it needs to regenerate this method + # In order to remove references to the original class when it is replaced + __annotate__.__generated_by_dataclasses__ = True + __annotate__.__qualname__ = f"{__class__.__qualname__}.{method_name}.__annotate__" + + return __annotate__ + + def _field_assign(frozen, name, value, self_name): # If we're a frozen class, then assign to our fields in __init__ # via object.__setattr__. Otherwise, just use a simple @@ -604,7 +664,7 @@ def _init_param(f): elif f.default_factory is not MISSING: # There's a factory function. Set a marker. default = '=__dataclass_HAS_DEFAULT_FACTORY__' - return f'{f.name}:__dataclass_type_{f.name}__{default}' + return f'{f.name}{default}' def _init_fn(fields, std_fields, kw_only_fields, frozen, has_post_init, @@ -627,11 +687,10 @@ def _init_fn(fields, std_fields, kw_only_fields, frozen, has_post_init, raise TypeError(f'non-default argument {f.name!r} ' f'follows default argument {seen_default.name!r}') - locals = {**{f'__dataclass_type_{f.name}__': f.type for f in fields}, - **{'__dataclass_HAS_DEFAULT_FACTORY__': _HAS_DEFAULT_FACTORY, - '__dataclass_builtins_object__': object, - } - } + annotation_fields = [f.name for f in fields if f.init] + + locals = {'__dataclass_HAS_DEFAULT_FACTORY__': _HAS_DEFAULT_FACTORY, + '__dataclass_builtins_object__': object} body_lines = [] for f in fields: @@ -655,14 +714,15 @@ def _init_fn(fields, std_fields, kw_only_fields, frozen, has_post_init, if kw_only_fields: # Add the keyword-only args. Because the * can only be added if # there's at least one keyword-only arg, there needs to be a test here - # (instead of just concatenting the lists together). + # (instead of just concatenating the lists together). _init_params += ['*'] _init_params += [_init_param(f) for f in kw_only_fields] func_builder.add_fn('__init__', [self_name] + _init_params, body_lines, locals=locals, - return_type=None) + return_type=None, + annotation_fields=annotation_fields) def _frozen_get_del_attr(cls, fields, func_builder): @@ -689,11 +749,8 @@ def _frozen_get_del_attr(cls, fields, func_builder): def _is_classvar(a_type, typing): - # This test uses a typing internal class, but it's the best way to - # test if this is a ClassVar. return (a_type is typing.ClassVar - or (type(a_type) is typing._GenericAlias - and a_type.__origin__ is typing.ClassVar)) + or (typing.get_origin(a_type) is typing.ClassVar)) def _is_initvar(a_type, dataclasses): @@ -981,7 +1038,8 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, # actual default value. Pseudo-fields ClassVars and InitVars are # included, despite the fact that they're not real fields. That's # dealt with later. - cls_annotations = inspect.get_annotations(cls) + cls_annotations = annotationlib.get_annotations( + cls, format=annotationlib.Format.FORWARDREF) # Now find fields in our class. While doing so, validate some # things, and set the default values (as class attributes) where @@ -1161,7 +1219,10 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, try: # In some cases fetching a signature is not possible. # But, we surely should not fail in this case. - text_sig = str(inspect.signature(cls)).replace(' -> None', '') + text_sig = str(inspect.signature( + cls, + annotation_format=annotationlib.Format.FORWARDREF, + )).replace(' -> None', '') except (TypeError, ValueError): text_sig = '' cls.__doc__ = (cls.__name__ + text_sig) @@ -1175,7 +1236,7 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, if weakref_slot and not slots: raise TypeError('weakref_slot is True but slots is False') if slots: - cls = _add_slots(cls, frozen, weakref_slot) + cls = _add_slots(cls, frozen, weakref_slot, fields) abc.update_abstractmethods(cls) @@ -1219,14 +1280,65 @@ def _get_slots(cls): raise TypeError(f"Slots of '{cls.__name__}' cannot be determined") -def _add_slots(cls, is_frozen, weakref_slot): - # Need to create a new class, since we can't set __slots__ - # after a class has been created. +def _update_func_cell_for__class__(f, oldcls, newcls): + # Returns True if we update a cell, else False. + if f is None: + # f will be None in the case of a property where not all of + # fget, fset, and fdel are used. Nothing to do in that case. + return False + try: + idx = f.__code__.co_freevars.index("__class__") + except ValueError: + # This function doesn't reference __class__, so nothing to do. + return False + # Fix the cell to point to the new class, if it's already pointing + # at the old class. I'm not convinced that the "is oldcls" test + # is needed, but other than performance can't hurt. + closure = f.__closure__[idx] + if closure.cell_contents is oldcls: + closure.cell_contents = newcls + return True + return False + + +def _create_slots(defined_fields, inherited_slots, field_names, weakref_slot): + # The slots for our class. Remove slots from our base classes. Add + # '__weakref__' if weakref_slot was given, unless it is already present. + seen_docs = False + slots = {} + for slot in itertools.filterfalse( + inherited_slots.__contains__, + itertools.chain( + # gh-93521: '__weakref__' also needs to be filtered out if + # already present in inherited_slots + field_names, ('__weakref__',) if weakref_slot else () + ) + ): + doc = getattr(defined_fields.get(slot), 'doc', None) + if doc is not None: + seen_docs = True + slots[slot] = doc + + # We only return dict if there's at least one doc member, + # otherwise we return tuple, which is the old default format. + if seen_docs: + return slots + return tuple(slots) + + +def _add_slots(cls, is_frozen, weakref_slot, defined_fields): + # Need to create a new class, since we can't set __slots__ after a + # class has been created, and the @dataclass decorator is called + # after the class is created. # Make sure __slots__ isn't already set. if '__slots__' in cls.__dict__: raise TypeError(f'{cls.__name__} already specifies __slots__') + # gh-102069: Remove existing __weakref__ descriptor. + # gh-135228: Make sure the original class can be garbage collected. + sys._clear_type_descriptors(cls) + # Create a new dict for our new class. cls_dict = dict(cls.__dict__) field_names = tuple(f.name for f in fields(cls)) @@ -1234,17 +1346,9 @@ def _add_slots(cls, is_frozen, weakref_slot): inherited_slots = set( itertools.chain.from_iterable(map(_get_slots, cls.__mro__[1:-1])) ) - # The slots for our class. Remove slots from our base classes. Add - # '__weakref__' if weakref_slot was given, unless it is already present. - cls_dict["__slots__"] = tuple( - itertools.filterfalse( - inherited_slots.__contains__, - itertools.chain( - # gh-93521: '__weakref__' also needs to be filtered out if - # already present in inherited_slots - field_names, ('__weakref__',) if weakref_slot else () - ) - ), + + cls_dict["__slots__"] = _create_slots( + defined_fields, inherited_slots, field_names, weakref_slot, ) for field_name in field_names: @@ -1252,26 +1356,59 @@ def _add_slots(cls, is_frozen, weakref_slot): # available in _MARKER. cls_dict.pop(field_name, None) - # Remove __dict__ itself. - cls_dict.pop('__dict__', None) - - # Clear existing `__weakref__` descriptor, it belongs to a previous type: - cls_dict.pop('__weakref__', None) # gh-102069 - # And finally create the class. qualname = getattr(cls, '__qualname__', None) - cls = type(cls)(cls.__name__, cls.__bases__, cls_dict) + newcls = type(cls)(cls.__name__, cls.__bases__, cls_dict) if qualname is not None: - cls.__qualname__ = qualname + newcls.__qualname__ = qualname if is_frozen: # Need this for pickling frozen classes with slots. if '__getstate__' not in cls_dict: - cls.__getstate__ = _dataclass_getstate + newcls.__getstate__ = _dataclass_getstate if '__setstate__' not in cls_dict: - cls.__setstate__ = _dataclass_setstate + newcls.__setstate__ = _dataclass_setstate + + # Fix up any closures which reference __class__. This is used to + # fix zero argument super so that it points to the correct class + # (the newly created one, which we're returning) and not the + # original class. We can break out of this loop as soon as we + # make an update, since all closures for a class will share a + # given cell. + for member in newcls.__dict__.values(): + # If this is a wrapped function, unwrap it. + member = inspect.unwrap(member) + + if isinstance(member, types.FunctionType): + if _update_func_cell_for__class__(member, cls, newcls): + break + elif isinstance(member, property): + if (_update_func_cell_for__class__(member.fget, cls, newcls) + or _update_func_cell_for__class__(member.fset, cls, newcls) + or _update_func_cell_for__class__(member.fdel, cls, newcls)): + break + + # Get new annotations to remove references to the original class + # in forward references + newcls_ann = annotationlib.get_annotations( + newcls, format=annotationlib.Format.FORWARDREF) + + # Fix references in dataclass Fields + for f in getattr(newcls, _FIELDS).values(): + try: + ann = newcls_ann[f.name] + except KeyError: + pass + else: + f.type = ann - return cls + # Fix the class reference in the __annotate__ method + init = newcls.__init__ + if init_annotate := getattr(init, "__annotate__", None): + if getattr(init_annotate, "__generated_by_dataclasses__", False): + _update_func_cell_for__class__(init_annotate, cls, newcls) + + return newcls def dataclass(cls=None, /, *, init=True, repr=True, eq=True, order=False, @@ -1490,7 +1627,7 @@ def _astuple_inner(obj, tuple_factory): def make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, match_args=True, kw_only=False, slots=False, - weakref_slot=False, module=None): + weakref_slot=False, module=None, decorator=dataclass): """Return a new dynamically created dataclass. The dataclass name will be 'cls_name'. 'fields' is an iterable @@ -1528,7 +1665,7 @@ class C(Base): for item in fields: if isinstance(item, str): name = item - tp = 'typing.Any' + tp = _ANY_MARKER elif len(item) == 2: name, tp, = item elif len(item) == 3: @@ -1547,15 +1684,49 @@ class C(Base): seen.add(name) annotations[name] = tp + # We initially block the VALUE format, because inside dataclass() we'll + # call get_annotations(), which will try the VALUE format first. If we don't + # block, that means we'd always end up eagerly importing typing here, which + # is what we're trying to avoid. + value_blocked = True + + def annotate_method(format): + def get_any(): + match format: + case annotationlib.Format.STRING: + return 'typing.Any' + case annotationlib.Format.FORWARDREF: + typing = sys.modules.get("typing") + if typing is None: + return annotationlib.ForwardRef("Any", module="typing") + else: + return typing.Any + case annotationlib.Format.VALUE: + if value_blocked: + raise NotImplementedError + from typing import Any + return Any + case _: + raise NotImplementedError + annos = { + ann: get_any() if t is _ANY_MARKER else t + for ann, t in annotations.items() + } + if format == annotationlib.Format.STRING: + return annotationlib.annotations_to_string(annos) + else: + return annos + # Update 'ns' with the user-supplied namespace plus our calculated values. def exec_body_callback(ns): ns.update(namespace) ns.update(defaults) - ns['__annotations__'] = annotations # We use `types.new_class()` instead of simply `type()` to allow dynamic creation # of generic dataclasses. cls = types.new_class(cls_name, bases, {}, exec_body_callback) + # For now, set annotations including the _ANY_MARKER. + cls.__annotate__ = annotate_method # For pickling to work, the __module__ variable needs to be set to the frame # where the dataclass is created. @@ -1570,11 +1741,14 @@ def exec_body_callback(ns): if module is not None: cls.__module__ = module - # Apply the normal decorator. - return dataclass(cls, init=init, repr=repr, eq=eq, order=order, - unsafe_hash=unsafe_hash, frozen=frozen, - match_args=match_args, kw_only=kw_only, slots=slots, - weakref_slot=weakref_slot) + # Apply the normal provided decorator. + cls = decorator(cls, init=init, repr=repr, eq=eq, order=order, + unsafe_hash=unsafe_hash, frozen=frozen, + match_args=match_args, kw_only=kw_only, slots=slots, + weakref_slot=weakref_slot) + # Now that the class is ready, allow the VALUE format. + value_blocked = False + return cls def replace(obj, /, **changes): diff --git a/Lib/datetime.py b/Lib/datetime.py index a33d2d724cb..14f30556584 100644 --- a/Lib/datetime.py +++ b/Lib/datetime.py @@ -1,9 +1,13 @@ +"""Specific date/time and related types. + +See https://data.iana.org/time-zones/tz-link.html for +time zone and DST data sources. +""" + try: from _datetime import * - from _datetime import __doc__ except ImportError: from _pydatetime import * - from _pydatetime import __doc__ __all__ = ("date", "datetime", "time", "timedelta", "timezone", "tzinfo", "MINYEAR", "MAXYEAR", "UTC") diff --git a/Lib/decimal.py b/Lib/decimal.py index ee3147f5dde..530bdfb3895 100644 --- a/Lib/decimal.py +++ b/Lib/decimal.py @@ -100,8 +100,8 @@ try: from _decimal import * - from _decimal import __version__ - from _decimal import __libmpdec_version__ + from _decimal import __version__ # noqa: F401 + from _decimal import __libmpdec_version__ # noqa: F401 except ImportError: import _pydecimal import sys diff --git a/Lib/difflib.py b/Lib/difflib.py index 33e7e6c165a..ac1ba4a6e4e 100644 --- a/Lib/difflib.py +++ b/Lib/difflib.py @@ -78,8 +78,8 @@ class SequenceMatcher: sequences. As a rule of thumb, a .ratio() value over 0.6 means the sequences are close matches: - >>> print(round(s.ratio(), 3)) - 0.866 + >>> print(round(s.ratio(), 2)) + 0.87 >>> If you're only interested in where the sequences match, @@ -908,87 +908,85 @@ def _fancy_replace(self, a, alo, ahi, b, blo, bhi): + abcdefGhijkl ? ^ ^ ^ """ - - # don't synch up unless the lines have a similarity score of at - # least cutoff; best_ratio tracks the best score seen so far - best_ratio, cutoff = 0.74, 0.75 + # Don't synch up unless the lines have a similarity score above + # cutoff. Previously only the smallest pair was handled here, + # and if there are many pairs with the best ratio, recursion + # could grow very deep, and runtime cubic. See: + # https://github.com/python/cpython/issues/119105 + # + # Later, more pathological cases prompted removing recursion + # entirely. + cutoff = 0.74999 cruncher = SequenceMatcher(self.charjunk) - eqi, eqj = None, None # 1st indices of equal lines (if any) + crqr = cruncher.real_quick_ratio + cqr = cruncher.quick_ratio + cr = cruncher.ratio - # search for the pair that matches best without being identical - # (identical lines must be junk lines, & we don't want to synch up - # on junk -- unless we have to) + WINDOW = 10 + best_i = best_j = None + dump_i, dump_j = alo, blo # smallest indices not yet resolved for j in range(blo, bhi): - bj = b[j] - cruncher.set_seq2(bj) - for i in range(alo, ahi): - ai = a[i] - if ai == bj: - if eqi is None: - eqi, eqj = i, j - continue - cruncher.set_seq1(ai) - # computing similarity is expensive, so use the quick - # upper bounds first -- have seen this speed up messy - # compares by a factor of 3. - # note that ratio() is only expensive to compute the first - # time it's called on a sequence pair; the expensive part - # of the computation is cached by cruncher - if cruncher.real_quick_ratio() > best_ratio and \ - cruncher.quick_ratio() > best_ratio and \ - cruncher.ratio() > best_ratio: - best_ratio, best_i, best_j = cruncher.ratio(), i, j - if best_ratio < cutoff: - # no non-identical "pretty close" pair - if eqi is None: - # no identical pair either -- treat it as a straight replace - yield from self._plain_replace(a, alo, ahi, b, blo, bhi) - return - # no close pair, but an identical pair -- synch up on that - best_i, best_j, best_ratio = eqi, eqj, 1.0 - else: - # there's a close pair, so forget the identical pair (if any) - eqi = None - - # a[best_i] very similar to b[best_j]; eqi is None iff they're not - # identical - - # pump out diffs from before the synch point - yield from self._fancy_helper(a, alo, best_i, b, blo, best_j) - - # do intraline marking on the synch pair - aelt, belt = a[best_i], b[best_j] - if eqi is None: - # pump out a '-', '?', '+', '?' quad for the synched lines - atags = btags = "" - cruncher.set_seqs(aelt, belt) - for tag, ai1, ai2, bj1, bj2 in cruncher.get_opcodes(): - la, lb = ai2 - ai1, bj2 - bj1 - if tag == 'replace': - atags += '^' * la - btags += '^' * lb - elif tag == 'delete': - atags += '-' * la - elif tag == 'insert': - btags += '+' * lb - elif tag == 'equal': - atags += ' ' * la - btags += ' ' * lb - else: - raise ValueError('unknown tag %r' % (tag,)) - yield from self._qformat(aelt, belt, atags, btags) - else: - # the synch pair is identical - yield ' ' + aelt + cruncher.set_seq2(b[j]) + # Search the corresponding i's within WINDOW for rhe highest + # ratio greater than `cutoff`. + aequiv = alo + (j - blo) + arange = range(max(aequiv - WINDOW, dump_i), + min(aequiv + WINDOW + 1, ahi)) + if not arange: # likely exit if `a` is shorter than `b` + break + best_ratio = cutoff + for i in arange: + cruncher.set_seq1(a[i]) + # Ordering by cheapest to most expensive ratio is very + # valuable, most often getting out early. + if (crqr() > best_ratio + and cqr() > best_ratio + and cr() > best_ratio): + best_i, best_j, best_ratio = i, j, cr() + + if best_i is None: + # found nothing to synch on yet - move to next j + continue - # pump out diffs from after the synch point - yield from self._fancy_helper(a, best_i+1, ahi, b, best_j+1, bhi) + # pump out straight replace from before this synch pair + yield from self._fancy_helper(a, dump_i, best_i, + b, dump_j, best_j) + # do intraline marking on the synch pair + aelt, belt = a[best_i], b[best_j] + if aelt != belt: + # pump out a '-', '?', '+', '?' quad for the synched lines + atags = btags = "" + cruncher.set_seqs(aelt, belt) + for tag, ai1, ai2, bj1, bj2 in cruncher.get_opcodes(): + la, lb = ai2 - ai1, bj2 - bj1 + if tag == 'replace': + atags += '^' * la + btags += '^' * lb + elif tag == 'delete': + atags += '-' * la + elif tag == 'insert': + btags += '+' * lb + elif tag == 'equal': + atags += ' ' * la + btags += ' ' * lb + else: + raise ValueError('unknown tag %r' % (tag,)) + yield from self._qformat(aelt, belt, atags, btags) + else: + # the synch pair is identical + yield ' ' + aelt + dump_i, dump_j = best_i + 1, best_j + 1 + best_i = best_j = None + + # pump out straight replace from after the last synch pair + yield from self._fancy_helper(a, dump_i, ahi, + b, dump_j, bhi) def _fancy_helper(self, a, alo, ahi, b, blo, bhi): g = [] if alo < ahi: if blo < bhi: - g = self._fancy_replace(a, alo, ahi, b, blo, bhi) + g = self._plain_replace(a, alo, ahi, b, blo, bhi) else: g = self._dump('-', a, alo, ahi) elif blo < bhi: @@ -1040,11 +1038,9 @@ def _qformat(self, aline, bline, atags, btags): # remaining is that perhaps it was really the case that " volatile" # was inserted after "private". I can live with that . -import re - -def IS_LINE_JUNK(line, pat=re.compile(r"\s*(?:#\s*)?$").match): +def IS_LINE_JUNK(line, pat=None): r""" - Return True for ignorable line: iff `line` is blank or contains a single '#'. + Return True for ignorable line: if `line` is blank or contains a single '#'. Examples: @@ -1056,6 +1052,11 @@ def IS_LINE_JUNK(line, pat=re.compile(r"\s*(?:#\s*)?$").match): False """ + if pat is None: + # Default: match '#' or the empty string + return line.strip() in '#' + # Previous versions used the undocumented parameter 'pat' as a + # match function. Retain this behaviour for compatibility. return pat(line) is not None def IS_CHARACTER_JUNK(ch, ws=" \t"): @@ -1266,6 +1267,12 @@ def _check_types(a, b, *args): if b and not isinstance(b[0], str): raise TypeError('lines to compare must be str, not %s (%r)' % (type(b[0]).__name__, b[0])) + if isinstance(a, str): + raise TypeError('input must be a sequence of strings, not %s' % + type(a).__name__) + if isinstance(b, str): + raise TypeError('input must be a sequence of strings, not %s' % + type(b).__name__) for arg in args: if not isinstance(arg, str): raise TypeError('all arguments must be str, not: %r' % (arg,)) @@ -1628,13 +1635,22 @@ def _line_pair_iterator(): """ _styles = """ + :root {color-scheme: light dark} table.diff {font-family: Menlo, Consolas, Monaco, Liberation Mono, Lucida Console, monospace; border:medium} .diff_header {background-color:#e0e0e0} td.diff_header {text-align:right} .diff_next {background-color:#c0c0c0} - .diff_add {background-color:#aaffaa} + .diff_add {background-color:palegreen} .diff_chg {background-color:#ffff77} - .diff_sub {background-color:#ffaaaa}""" + .diff_sub {background-color:#ffaaaa} + + @media (prefers-color-scheme: dark) { + .diff_header {background-color:#666} + .diff_next {background-color:#393939} + .diff_add {background-color:darkgreen} + .diff_chg {background-color:#847415} + .diff_sub {background-color:darkred} + }""" _table_template = """ '). \ replace('\t',' ') -del re def restore(delta, which): r""" @@ -2047,10 +2062,3 @@ def restore(delta, which): for line in delta: if line[:2] in prefixes: yield line[2:] - -def _test(): - import doctest, difflib - return doctest.testmod(difflib) - -if __name__ == "__main__": - _test() diff --git a/Lib/doctest.py b/Lib/doctest.py index ecac54ad5a5..a66888d8fc9 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -109,6 +109,8 @@ def _test(): from _colorize import ANSIColors, can_colorize +__unittest = True + class TestResults(namedtuple('TestResults', 'failed attempted')): def __new__(cls, failed, attempted, *, skipped=0): results = super().__new__(cls, failed, attempted) @@ -390,11 +392,11 @@ def __init__(self, out): # still use input() to get user input self.use_rawinput = 1 - def set_trace(self, frame=None): + def set_trace(self, frame=None, *, commands=None): self.__debugger_used = True if frame is None: frame = sys._getframe().f_back - pdb.Pdb.set_trace(self, frame) + pdb.Pdb.set_trace(self, frame, commands=commands) def set_continue(self): # Calling set_continue unconditionally would break unit test @@ -1230,7 +1232,7 @@ class DocTestRunner: `OutputChecker` to the constructor. The test runner's display output can be controlled in two ways. - First, an output function (`out) can be passed to + First, an output function (`out`) can be passed to `TestRunner.run`; this function will be called with strings that should be displayed. It defaults to `sys.stdout.write`. If capturing the output is not sufficient, then the display output @@ -1398,11 +1400,11 @@ def __run(self, test, compileflags, out): exec(compile(example.source, filename, "single", compileflags, True), test.globs) self.debugger.set_continue() # ==== Example Finished ==== - exception = None + exc_info = None except KeyboardInterrupt: raise - except: - exception = sys.exc_info() + except BaseException as exc: + exc_info = type(exc), exc, exc.__traceback__.tb_next self.debugger.set_continue() # ==== Example Finished ==== got = self._fakeout.getvalue() # the actual output @@ -1411,21 +1413,21 @@ def __run(self, test, compileflags, out): # If the example executed without raising any exceptions, # verify its output. - if exception is None: + if exc_info is None: if check(example.want, got, self.optionflags): outcome = SUCCESS # The example raised an exception: check if it was expected. else: - formatted_ex = traceback.format_exception_only(*exception[:2]) - if issubclass(exception[0], SyntaxError): + formatted_ex = traceback.format_exception_only(*exc_info[:2]) + if issubclass(exc_info[0], SyntaxError): # SyntaxError / IndentationError is special: # we don't care about the carets / suggestions / etc # We only care about the error message and notes. # They start with `SyntaxError:` (or any other class name) exception_line_prefixes = ( - f"{exception[0].__qualname__}:", - f"{exception[0].__module__}.{exception[0].__qualname__}:", + f"{exc_info[0].__qualname__}:", + f"{exc_info[0].__module__}.{exc_info[0].__qualname__}:", ) exc_msg_index = next( index @@ -1436,7 +1438,7 @@ def __run(self, test, compileflags, out): exc_msg = "".join(formatted_ex) if not quiet: - got += _exception_traceback(exception) + got += _exception_traceback(exc_info) # If `example.exc_msg` is None, then we weren't expecting # an exception. @@ -1465,7 +1467,7 @@ def __run(self, test, compileflags, out): elif outcome is BOOM: if not quiet: self.report_unexpected_exception(out, test, example, - exception) + exc_info) failures += 1 else: assert False, ("unknown outcome", outcome) @@ -2327,7 +2329,7 @@ def runTest(self): sys.stdout = old if results.failed: - raise self.failureException(self.format_failure(new.getvalue())) + raise self.failureException(self.format_failure(new.getvalue().rstrip('\n'))) def format_failure(self, err): test = self._dt_test @@ -2737,7 +2739,7 @@ def testsource(module, name): return testsrc def debug_src(src, pm=False, globs=None): - """Debug a single doctest docstring, in argument `src`'""" + """Debug a single doctest docstring, in argument `src`""" testsrc = script_from_examples(src) debug_script(testsrc, pm, globs) @@ -2873,7 +2875,7 @@ def get(self): def _test(): import argparse - parser = argparse.ArgumentParser(description="doctest runner") + parser = argparse.ArgumentParser(description="doctest runner", color=True) parser.add_argument('-v', '--verbose', action='store_true', default=False, help='print very verbose output for all tests') parser.add_argument('-o', '--option', action='append', diff --git a/Lib/encodings/__init__.py b/Lib/encodings/__init__.py index f9075b8f0d9..298177eb800 100644 --- a/Lib/encodings/__init__.py +++ b/Lib/encodings/__init__.py @@ -156,19 +156,22 @@ def search_function(encoding): codecs.register(search_function) if sys.platform == 'win32': - # bpo-671666, bpo-46668: If Python does not implement a codec for current - # Windows ANSI code page, use the "mbcs" codec instead: - # WideCharToMultiByte() and MultiByteToWideChar() functions with CP_ACP. - # Python does not support custom code pages. - def _alias_mbcs(encoding): + from ._win_cp_codecs import create_win32_code_page_codec + + def win32_code_page_search_function(encoding): + encoding = encoding.lower() + if not encoding.startswith('cp'): + return None try: - import _winapi - ansi_code_page = "cp%s" % _winapi.GetACP() - if encoding == ansi_code_page: - import encodings.mbcs - return encodings.mbcs.getregentry() - except ImportError: - # Imports may fail while we are shutting down - pass + cp = int(encoding[2:]) + except ValueError: + return None + # Test if the code page is supported + try: + codecs.code_page_encode(cp, 'x') + except (OverflowError, OSError): + return None + + return create_win32_code_page_codec(cp) - codecs.register(_alias_mbcs) + codecs.register(win32_code_page_search_function) diff --git a/Lib/encodings/_win_cp_codecs.py b/Lib/encodings/_win_cp_codecs.py new file mode 100644 index 00000000000..4f8eb886794 --- /dev/null +++ b/Lib/encodings/_win_cp_codecs.py @@ -0,0 +1,36 @@ +import codecs + +def create_win32_code_page_codec(cp): + from codecs import code_page_encode, code_page_decode + + def encode(input, errors='strict'): + return code_page_encode(cp, input, errors) + + def decode(input, errors='strict'): + return code_page_decode(cp, input, errors, True) + + class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input, final=False): + return code_page_encode(cp, input, self.errors)[0] + + class IncrementalDecoder(codecs.BufferedIncrementalDecoder): + def _buffer_decode(self, input, errors, final): + return code_page_decode(cp, input, errors, final) + + class StreamWriter(codecs.StreamWriter): + def encode(self, input, errors='strict'): + return code_page_encode(cp, input, errors) + + class StreamReader(codecs.StreamReader): + def decode(self, input, errors, final): + return code_page_decode(cp, input, errors, final) + + return codecs.CodecInfo( + name=f'cp{cp}', + encode=encode, + decode=decode, + incrementalencoder=IncrementalEncoder, + incrementaldecoder=IncrementalDecoder, + streamreader=StreamReader, + streamwriter=StreamWriter, + ) diff --git a/Lib/encodings/aliases.py b/Lib/encodings/aliases.py index 6a5ca046b5e..4ecb6b6e297 100644 --- a/Lib/encodings/aliases.py +++ b/Lib/encodings/aliases.py @@ -204,6 +204,11 @@ 'csibm869' : 'cp869', 'ibm869' : 'cp869', + # cp874 codec + '874' : 'cp874', + 'ms874' : 'cp874', + 'windows_874' : 'cp874', + # cp932 codec '932' : 'cp932', 'ms932' : 'cp932', @@ -241,6 +246,7 @@ 'ks_c_5601_1987' : 'euc_kr', 'ksx1001' : 'euc_kr', 'ks_x_1001' : 'euc_kr', + 'cseuckr' : 'euc_kr', # gb18030 codec 'gb18030_2000' : 'gb18030', @@ -399,6 +405,8 @@ 'iso_8859_8' : 'iso8859_8', 'iso_8859_8_1988' : 'iso8859_8', 'iso_ir_138' : 'iso8859_8', + 'iso_8859_8_i' : 'iso8859_8', + 'iso_8859_8_e' : 'iso8859_8', # iso8859_9 codec 'csisolatin5' : 'iso8859_9', diff --git a/Lib/enum.py b/Lib/enum.py index 3adb208c7e6..b4551da1c17 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -1,6 +1,5 @@ import sys import builtins as bltns -from functools import partial from types import MappingProxyType, DynamicClassAttribute @@ -38,7 +37,7 @@ def _is_descriptor(obj): """ Returns True if obj is a descriptor, False otherwise. """ - return not isinstance(obj, partial) and ( + return ( hasattr(obj, '__get__') or hasattr(obj, '__set__') or hasattr(obj, '__delete__') @@ -130,7 +129,7 @@ def show_flag_values(value): def bin(num, max_bits=None): """ Like built-in bin(), except negative values are represented in - twos-compliment, and the leading bit always indicates sign + twos-complement, and the leading bit always indicates sign (0=positive, 1=negative). >>> bin(10) @@ -139,6 +138,7 @@ def bin(num, max_bits=None): '0b1 0101' """ + num = num.__index__() ceiling = 2 ** (num).bit_length() if num >= 0: s = bltns.bin(num + ceiling).replace('1', '0', 1) @@ -151,18 +151,6 @@ def bin(num, max_bits=None): digits = (sign[-1] * max_bits + digits)[-max_bits:] return "%s %s" % (sign, digits) -def _dedent(text): - """ - Like textwrap.dedent. Rewritten because we cannot import textwrap. - """ - lines = text.split('\n') - for i, ch in enumerate(lines[0]): - if ch != ' ': - break - for j, l in enumerate(lines): - lines[j] = l[i:] - return '\n'.join(lines) - class _not_given: def __repr__(self): return('') @@ -208,7 +196,7 @@ def __get__(self, instance, ownerclass=None): # use previous enum.property return self.fget(instance) elif self._attr_type == 'attr': - # look up previous attibute + # look up previous attribute return getattr(self._cls_type, self.name) elif self._attr_type == 'desc': # use previous descriptor @@ -406,12 +394,6 @@ def __setitem__(self, key, value): elif isinstance(value, nonmember): # unwrap value here; it won't be processed by the below `else` value = value.value - elif isinstance(value, partial): - import warnings - warnings.warn('functools.partial will be a method descriptor ' - 'in future Python versions; wrap it in ' - 'enum.member() if you want to preserve the ' - 'old behavior', FutureWarning, stacklevel=2) elif _is_descriptor(value): pass elif self._cls_name is not None and _is_internal_class(self._cls_name, value): @@ -1103,6 +1085,21 @@ def _add_member_(cls, name, member): # now add to _member_map_ (even aliases) cls._member_map_[name] = member + @property + def __signature__(cls): + from inspect import Parameter, Signature + if cls._member_names_: + return Signature([Parameter('values', Parameter.VAR_POSITIONAL)]) + else: + return Signature([Parameter('new_class_name', Parameter.POSITIONAL_ONLY), + Parameter('names', Parameter.POSITIONAL_OR_KEYWORD), + Parameter('module', Parameter.KEYWORD_ONLY, default=None), + Parameter('qualname', Parameter.KEYWORD_ONLY, default=None), + Parameter('type', Parameter.KEYWORD_ONLY, default=None), + Parameter('start', Parameter.KEYWORD_ONLY, default=1), + Parameter('boundary', Parameter.KEYWORD_ONLY, default=None)]) + + EnumMeta = EnumType # keep EnumMeta name for backwards compatibility @@ -1146,13 +1143,6 @@ class Enum(metaclass=EnumType): attributes -- see the documentation for details. """ - @classmethod - def __signature__(cls): - if cls._member_names_: - return '(*values)' - else: - return '(new_class_name, /, names, *, module=None, qualname=None, type=None, start=1, boundary=None)' - def __new__(cls, value): # all enum instances are actually created during class construction # without calling this method; this method is called by the metaclass' @@ -1214,9 +1204,6 @@ def __new__(cls, value): exc = None ve_exc = None - def __init__(self, *args, **kwds): - pass - def _add_alias_(self, name): self.__class__._add_member_(name, self) @@ -2041,8 +2028,7 @@ def _test_simple_enum(checked_enum, simple_enum): ... RED = auto() ... GREEN = auto() ... BLUE = auto() - ... # TODO: RUSTPYTHON - >>> _test_simple_enum(CheckedColor, Color) # doctest: +SKIP + >>> _test_simple_enum(CheckedColor, Color) If differences are found, a :exc:`TypeError` is raised. """ diff --git a/Lib/fnmatch.py b/Lib/fnmatch.py index 73acb1fe8d4..10e1c936688 100644 --- a/Lib/fnmatch.py +++ b/Lib/fnmatch.py @@ -9,12 +9,15 @@ The function translate(PATTERN) returns a regular expression corresponding to PATTERN. (It does not compile it.) """ + +import functools +import itertools import os import posixpath import re -import functools -__all__ = ["filter", "fnmatch", "fnmatchcase", "translate"] +__all__ = ["filter", "filterfalse", "fnmatch", "fnmatchcase", "translate"] + def fnmatch(name, pat): """Test whether FILENAME matches PATTERN. @@ -35,6 +38,7 @@ def fnmatch(name, pat): pat = os.path.normcase(pat) return fnmatchcase(name, pat) + @functools.lru_cache(maxsize=32768, typed=True) def _compile_pattern(pat): if isinstance(pat, bytes): @@ -45,6 +49,7 @@ def _compile_pattern(pat): res = translate(pat) return re.compile(res).match + def filter(names, pat): """Construct a list from those elements of the iterable NAMES that match PAT.""" result = [] @@ -61,6 +66,22 @@ def filter(names, pat): result.append(name) return result + +def filterfalse(names, pat): + """Construct a list from those elements of the iterable NAMES that do not match PAT.""" + pat = os.path.normcase(pat) + match = _compile_pattern(pat) + if os.path is posixpath: + # normcase on posix is NOP. Optimize it away from the loop. + return list(itertools.filterfalse(match, names)) + + result = [] + for name in names: + if match(os.path.normcase(name)) is None: + result.append(name) + return result + + def fnmatchcase(name, pat): """Test whether FILENAME matches PATTERN, including case. @@ -77,24 +98,32 @@ def translate(pat): There is no way to quote meta-characters. """ - STAR = object() - parts = _translate(pat, STAR, '.') - return _join_translated_parts(parts, STAR) + parts, star_indices = _translate(pat, '*', '.') + return _join_translated_parts(parts, star_indices) + +_re_setops_sub = re.compile(r'([&~|])').sub +_re_escape = functools.lru_cache(maxsize=512)(re.escape) -def _translate(pat, STAR, QUESTION_MARK): + +def _translate(pat, star, question_mark): res = [] add = res.append + star_indices = [] + i, n = 0, len(pat) while i < n: c = pat[i] i = i+1 if c == '*': + # store the position of the wildcard + star_indices.append(len(res)) + add(star) # compress consecutive `*` into one - if (not res) or res[-1] is not STAR: - add(STAR) + while i < n and pat[i] == '*': + i += 1 elif c == '?': - add(QUESTION_MARK) + add(question_mark) elif c == '[': j = i if j < n and pat[j] == '!': @@ -133,8 +162,6 @@ def _translate(pat, STAR, QUESTION_MARK): # Hyphens that create ranges shouldn't be escaped. stuff = '-'.join(s.replace('\\', r'\\').replace('-', r'\-') for s in chunks) - # Escape set operations (&&, ~~ and ||). - stuff = re.sub(r'([&~|])', r'\\\1', stuff) i = j+1 if not stuff: # Empty range: never match. @@ -143,50 +170,40 @@ def _translate(pat, STAR, QUESTION_MARK): # Negated empty range: match any character. add('.') else: + # Escape set operations (&&, ~~ and ||). + stuff = _re_setops_sub(r'\\\1', stuff) if stuff[0] == '!': stuff = '^' + stuff[1:] elif stuff[0] in ('^', '['): stuff = '\\' + stuff add(f'[{stuff}]') else: - add(re.escape(c)) - assert i == n - return res - - -def _join_translated_parts(inp, STAR): - # Deal with STARs. - res = [] - add = res.append - i, n = 0, len(inp) - # Fixed pieces at the start? - while i < n and inp[i] is not STAR: - add(inp[i]) - i += 1 - # Now deal with STAR fixed STAR fixed ... - # For an interior `STAR fixed` pairing, we want to do a minimal - # .*? match followed by `fixed`, with no possibility of backtracking. - # Atomic groups ("(?>...)") allow us to spell that directly. - # Note: people rely on the undocumented ability to join multiple - # translate() results together via "|" to build large regexps matching - # "one of many" shell patterns. - while i < n: - assert inp[i] is STAR - i += 1 - if i == n: - add(".*") - break - assert inp[i] is not STAR - fixed = [] - while i < n and inp[i] is not STAR: - fixed.append(inp[i]) - i += 1 - fixed = "".join(fixed) - if i == n: - add(".*") - add(fixed) - else: - add(f"(?>.*?{fixed})") + add(_re_escape(c)) assert i == n - res = "".join(res) - return fr'(?s:{res})\Z' + return res, star_indices + + +def _join_translated_parts(parts, star_indices): + if not star_indices: + return fr'(?s:{"".join(parts)})\z' + iter_star_indices = iter(star_indices) + j = next(iter_star_indices) + buffer = parts[:j] # fixed pieces at the start + append, extend = buffer.append, buffer.extend + i = j + 1 + for j in iter_star_indices: + # Now deal with STAR fixed STAR fixed ... + # For an interior `STAR fixed` pairing, we want to do a minimal + # .*? match followed by `fixed`, with no possibility of backtracking. + # Atomic groups ("(?>...)") allow us to spell that directly. + # Note: people rely on the undocumented ability to join multiple + # translate() results together via "|" to build large regexps matching + # "one of many" shell patterns. + append('(?>.*?') + extend(parts[i:j]) + append(')') + i = j + 1 + append('.*') + extend(parts[i:]) + res = ''.join(buffer) + return fr'(?s:{res})\z' diff --git a/Lib/fractions.py b/Lib/fractions.py index 9d42e809875..a497ee19935 100644 --- a/Lib/fractions.py +++ b/Lib/fractions.py @@ -3,7 +3,6 @@ """Fraction, infinite-precision, rational numbers.""" -from decimal import Decimal import functools import math import numbers @@ -65,7 +64,7 @@ def _hash_algorithm(numerator, denominator): (?:\.(?P\d*|\d+(_\d+)*))? # an optional fractional part (?:E(?P[-+]?\d+(_\d+)*))? # and optional exponent ) - \s*\Z # and optional whitespace to finish + \s*\z # and optional whitespace to finish """, re.VERBOSE | re.IGNORECASE) @@ -169,9 +168,13 @@ def _round_to_figures(n, d, figures): # A '0' that's *not* followed by another digit is parsed as a minimum width # rather than a zeropad flag. (?P0(?=[0-9]))? - (?P0|[1-9][0-9]*)? + (?P[0-9]+)? (?P[,_])? - (?:\.(?P0|[1-9][0-9]*))? + (?:\. + (?=[,_0-9]) # lookahead for digit or separator + (?P[0-9]+)? + (?P[,_])? + )? (?P[eEfFgG%]) """, re.DOTALL | re.VERBOSE).fullmatch @@ -244,7 +247,9 @@ def __new__(cls, numerator=0, denominator=None): self._denominator = numerator.denominator return self - elif isinstance(numerator, (float, Decimal)): + elif (isinstance(numerator, float) or + (not isinstance(numerator, type) and + hasattr(numerator, 'as_integer_ratio'))): # Exact conversion self._numerator, self._denominator = numerator.as_integer_ratio() return self @@ -278,8 +283,8 @@ def __new__(cls, numerator=0, denominator=None): numerator = -numerator else: - raise TypeError("argument should be a string " - "or a Rational instance") + raise TypeError("argument should be a string or a Rational " + "instance or have the as_integer_ratio() method") elif type(numerator) is int is type(denominator): pass # *very* normal case @@ -305,6 +310,28 @@ def __new__(cls, numerator=0, denominator=None): self._denominator = denominator return self + @classmethod + def from_number(cls, number): + """Converts a finite real number to a rational number, exactly. + + Beware that Fraction.from_number(0.3) != Fraction(3, 10). + + """ + if type(number) is int: + return cls._from_coprime_ints(number, 1) + + elif isinstance(number, numbers.Rational): + return cls._from_coprime_ints(number.numerator, number.denominator) + + elif (isinstance(number, float) or + (not isinstance(number, type) and + hasattr(number, 'as_integer_ratio'))): + return cls._from_coprime_ints(*number.as_integer_ratio()) + + else: + raise TypeError("argument should be a Rational instance or " + "have the as_integer_ratio() method") + @classmethod def from_float(cls, f): """Converts a finite float to a rational number, exactly. @@ -476,6 +503,7 @@ def _format_float_style(self, match): minimumwidth = int(match["minimumwidth"] or "0") thousands_sep = match["thousands_sep"] precision = int(match["precision"] or "6") + frac_sep = match["frac_separators"] or "" presentation_type = match["presentation_type"] trim_zeros = presentation_type in "gG" and not alternate_form trim_point = not alternate_form @@ -532,6 +560,9 @@ def _format_float_style(self, match): if trim_zeros: frac_part = frac_part.rstrip("0") separator = "" if trim_point and not frac_part else "." + if frac_sep: + frac_part = frac_sep.join(frac_part[pos:pos + 3] + for pos in range(0, len(frac_part), 3)) trailing = separator + frac_part + suffix # Do zero padding if required. @@ -671,7 +702,7 @@ def forward(a, b): elif isinstance(b, float): return fallback_operator(float(a), b) elif handle_complex and isinstance(b, complex): - return fallback_operator(complex(a), b) + return fallback_operator(float(a), b) else: return NotImplemented forward.__name__ = '__' + fallback_operator.__name__ + '__' @@ -684,7 +715,7 @@ def reverse(b, a): elif isinstance(a, numbers.Real): return fallback_operator(float(a), float(b)) elif handle_complex and isinstance(a, numbers.Complex): - return fallback_operator(complex(a), complex(b)) + return fallback_operator(complex(a), float(b)) else: return NotImplemented reverse.__name__ = '__r' + fallback_operator.__name__ + '__' @@ -851,7 +882,7 @@ def _mod(a, b): __mod__, __rmod__ = _operator_fallbacks(_mod, operator.mod, False) - def __pow__(a, b): + def __pow__(a, b, modulo=None): """a ** b If b is not an integer, the result will be a float or complex @@ -859,6 +890,8 @@ def __pow__(a, b): result will be rational. """ + if modulo is not None: + return NotImplemented if isinstance(b, numbers.Rational): if b.denominator == 1: power = b.numerator @@ -883,8 +916,10 @@ def __pow__(a, b): else: return NotImplemented - def __rpow__(b, a): + def __rpow__(b, a, modulo=None): """a ** b""" + if modulo is not None: + return NotImplemented if b._denominator == 1 and b._numerator >= 0: # If a is an int, keep it that way if possible. return a ** b._numerator diff --git a/Lib/ftplib.py b/Lib/ftplib.py index 10c5d1ea08a..50771e8c17c 100644 --- a/Lib/ftplib.py +++ b/Lib/ftplib.py @@ -343,7 +343,7 @@ def ntransfercmd(self, cmd, rest=None): connection and the expected size of the transfer. The expected size may be None if it could not be determined. - Optional `rest' argument can be a string that is sent as the + Optional 'rest' argument can be a string that is sent as the argument to a REST command. This is essentially a server marker used to tell the server to skip over any data up to the given marker. diff --git a/Lib/getopt.py b/Lib/getopt.py index 5419d77f5d7..25f3e2439b3 100644 --- a/Lib/getopt.py +++ b/Lib/getopt.py @@ -2,8 +2,8 @@ This module helps scripts to parse the command line arguments in sys.argv. It supports the same conventions as the Unix getopt() -function (including the special meanings of arguments of the form `-' -and `--'). Long options similar to those supported by GNU software +function (including the special meanings of arguments of the form '-' +and '--'). Long options similar to those supported by GNU software may be used as well via an optional third argument. This module provides two functions and an exception: @@ -24,21 +24,14 @@ # TODO for gnu_getopt(): # # - GNU getopt_long_only mechanism -# - allow the caller to specify ordering -# - RETURN_IN_ORDER option -# - GNU extension with '-' as first character of option string -# - optional arguments, specified by double colons # - an option string with a W followed by semicolon should # treat "-W foo" as "--foo" __all__ = ["GetoptError","error","getopt","gnu_getopt"] import os -try: - from gettext import gettext as _ -except ImportError: - # Bootstrapping Python: gettext's dependencies not built yet - def _(s): return s +from gettext import gettext as _ + class GetoptError(Exception): opt = '' @@ -61,12 +54,14 @@ def getopt(args, shortopts, longopts = []): running program. Typically, this means "sys.argv[1:]". shortopts is the string of option letters that the script wants to recognize, with options that require an argument followed by a - colon (i.e., the same format that Unix getopt() uses). If + colon and options that accept an optional argument followed by + two colons (i.e., the same format that Unix getopt() uses). If specified, longopts is a list of strings with the names of the long options which should be supported. The leading '--' characters should not be included in the option name. Options which require an argument should be followed by an equal sign - ('='). + ('='). Options which accept an optional argument should be + followed by an equal sign and question mark ('=?'). The return value consists of two elements: the first is a list of (option, value) pairs; the second is the list of program arguments @@ -105,7 +100,7 @@ def gnu_getopt(args, shortopts, longopts = []): processing options as soon as a non-option argument is encountered. - If the first character of the option string is `+', or if the + If the first character of the option string is '+', or if the environment variable POSIXLY_CORRECT is set, then option processing stops as soon as a non-option argument is encountered. @@ -118,8 +113,13 @@ def gnu_getopt(args, shortopts, longopts = []): else: longopts = list(longopts) + return_in_order = False + if shortopts.startswith('-'): + shortopts = shortopts[1:] + all_options_first = False + return_in_order = True # Allow options after non-option arguments? - if shortopts.startswith('+'): + elif shortopts.startswith('+'): shortopts = shortopts[1:] all_options_first = True elif os.environ.get("POSIXLY_CORRECT"): @@ -133,8 +133,14 @@ def gnu_getopt(args, shortopts, longopts = []): break if args[0][:2] == '--': + if return_in_order and prog_args: + opts.append((None, prog_args)) + prog_args = [] opts, args = do_longs(opts, args[0][2:], longopts, args[1:]) elif args[0][:1] == '-' and args[0] != '-': + if return_in_order and prog_args: + opts.append((None, prog_args)) + prog_args = [] opts, args = do_shorts(opts, args[0][1:], shortopts, args[1:]) else: if all_options_first: @@ -156,7 +162,7 @@ def do_longs(opts, opt, longopts, args): has_arg, opt = long_has_args(opt, longopts) if has_arg: - if optarg is None: + if optarg is None and has_arg != '?': if not args: raise GetoptError(_('option --%s requires argument') % opt, opt) optarg, args = args[0], args[1:] @@ -177,13 +183,19 @@ def long_has_args(opt, longopts): return False, opt elif opt + '=' in possibilities: return True, opt - # No exact match, so better be unique. + elif opt + '=?' in possibilities: + return '?', opt + # Possibilities must be unique to be accepted if len(possibilities) > 1: - # XXX since possibilities contains all valid continuations, might be - # nice to work them into the error msg - raise GetoptError(_('option --%s not a unique prefix') % opt, opt) + raise GetoptError( + _("option --%s not a unique prefix; possible options: %s") + % (opt, ", ".join(possibilities)), + opt, + ) assert len(possibilities) == 1 unique_match = possibilities[0] + if unique_match.endswith('=?'): + return '?', unique_match[:-2] has_arg = unique_match.endswith('=') if has_arg: unique_match = unique_match[:-1] @@ -192,8 +204,9 @@ def long_has_args(opt, longopts): def do_shorts(opts, optstring, shortopts, args): while optstring != '': opt, optstring = optstring[0], optstring[1:] - if short_has_arg(opt, shortopts): - if optstring == '': + has_arg = short_has_arg(opt, shortopts) + if has_arg: + if optstring == '' and has_arg != '?': if not args: raise GetoptError(_('option -%s requires argument') % opt, opt) @@ -207,7 +220,11 @@ def do_shorts(opts, optstring, shortopts, args): def short_has_arg(opt, shortopts): for i in range(len(shortopts)): if opt == shortopts[i] != ':': - return shortopts.startswith(':', i+1) + if not shortopts.startswith(':', i+1): + return False + if shortopts.startswith('::', i+1): + return '?' + return True raise GetoptError(_('option -%s not recognized') % opt, opt) if __name__ == '__main__': diff --git a/Lib/getpass.py b/Lib/getpass.py index bd0097ced94..3d9bb1f0d14 100644 --- a/Lib/getpass.py +++ b/Lib/getpass.py @@ -1,6 +1,7 @@ """Utilities to get a password and/or the current user name. -getpass(prompt[, stream]) - Prompt for a password, with echo turned off. +getpass(prompt[, stream[, echo_char]]) - Prompt for a password, with echo +turned off and optional keyboard feedback. getuser() - Get the user name from the environment or password database. GetPassWarning - This UserWarning is issued when getpass() cannot prevent @@ -25,13 +26,15 @@ class GetPassWarning(UserWarning): pass -def unix_getpass(prompt='Password: ', stream=None): +def unix_getpass(prompt='Password: ', stream=None, *, echo_char=None): """Prompt for a password, with echo turned off. Args: prompt: Written on stream to ask for the input. Default: 'Password: ' stream: A writable file object to display the prompt. Defaults to the tty. If no tty is available defaults to sys.stderr. + echo_char: A single ASCII character to mask input (e.g., '*'). + If None, input is hidden. Returns: The seKr3t input. Raises: @@ -40,6 +43,8 @@ def unix_getpass(prompt='Password: ', stream=None): Always restores terminal settings before returning. """ + _check_echo_char(echo_char) + passwd = None with contextlib.ExitStack() as stack: try: @@ -68,12 +73,16 @@ def unix_getpass(prompt='Password: ', stream=None): old = termios.tcgetattr(fd) # a copy to save new = old[:] new[3] &= ~termios.ECHO # 3 == 'lflags' + if echo_char: + new[3] &= ~termios.ICANON tcsetattr_flags = termios.TCSAFLUSH if hasattr(termios, 'TCSASOFT'): tcsetattr_flags |= termios.TCSASOFT try: termios.tcsetattr(fd, tcsetattr_flags, new) - passwd = _raw_input(prompt, stream, input=input) + passwd = _raw_input(prompt, stream, input=input, + echo_char=echo_char) + finally: termios.tcsetattr(fd, tcsetattr_flags, old) stream.flush() # issue7208 @@ -93,10 +102,11 @@ def unix_getpass(prompt='Password: ', stream=None): return passwd -def win_getpass(prompt='Password: ', stream=None): +def win_getpass(prompt='Password: ', stream=None, *, echo_char=None): """Prompt for password with echo off, using Windows getwch().""" if sys.stdin is not sys.__stdin__: return fallback_getpass(prompt, stream) + _check_echo_char(echo_char) for c in prompt: msvcrt.putwch(c) @@ -108,25 +118,48 @@ def win_getpass(prompt='Password: ', stream=None): if c == '\003': raise KeyboardInterrupt if c == '\b': + if echo_char and pw: + msvcrt.putwch('\b') + msvcrt.putwch(' ') + msvcrt.putwch('\b') pw = pw[:-1] else: pw = pw + c + if echo_char: + msvcrt.putwch(echo_char) msvcrt.putwch('\r') msvcrt.putwch('\n') return pw -def fallback_getpass(prompt='Password: ', stream=None): +def fallback_getpass(prompt='Password: ', stream=None, *, echo_char=None): + _check_echo_char(echo_char) import warnings warnings.warn("Can not control echo on the terminal.", GetPassWarning, stacklevel=2) if not stream: stream = sys.stderr print("Warning: Password input may be echoed.", file=stream) - return _raw_input(prompt, stream) + return _raw_input(prompt, stream, echo_char=echo_char) + +def _check_echo_char(echo_char): + # Single-character ASCII excluding control characters + if echo_char is None: + return + if not isinstance(echo_char, str): + raise TypeError("'echo_char' must be a str or None, not " + f"{type(echo_char).__name__}") + if not ( + len(echo_char) == 1 + and echo_char.isprintable() + and echo_char.isascii() + ): + raise ValueError("'echo_char' must be a single printable ASCII " + f"character, got: {echo_char!r}") -def _raw_input(prompt="", stream=None, input=None): + +def _raw_input(prompt="", stream=None, input=None, echo_char=None): # This doesn't save the string in the GNU readline history. if not stream: stream = sys.stderr @@ -143,6 +176,8 @@ def _raw_input(prompt="", stream=None, input=None): stream.write(prompt) stream.flush() # NOTE: The Python C API calls flockfile() (and unlock) during readline. + if echo_char: + return _readline_with_echo_char(stream, input, echo_char) line = input.readline() if not line: raise EOFError @@ -151,6 +186,35 @@ def _raw_input(prompt="", stream=None, input=None): return line +def _readline_with_echo_char(stream, input, echo_char): + passwd = "" + eof_pressed = False + while True: + char = input.read(1) + if char == '\n' or char == '\r': + break + elif char == '\x03': + raise KeyboardInterrupt + elif char == '\x7f' or char == '\b': + if passwd: + stream.write("\b \b") + stream.flush() + passwd = passwd[:-1] + elif char == '\x04': + if eof_pressed: + break + else: + eof_pressed = True + elif char == '\x00': + continue + else: + passwd += char + stream.write(echo_char) + stream.flush() + eof_pressed = False + return passwd + + def getuser(): """Get the username from the environment or password database. diff --git a/Lib/gettext.py b/Lib/gettext.py index 62cff81b7b3..6c11ab2b1eb 100644 --- a/Lib/gettext.py +++ b/Lib/gettext.py @@ -41,14 +41,10 @@ # to do binary searches and lazy initializations. Or you might want to use # the undocumented double-hash algorithm for .mo files with hash tables, but # you'll need to study the GNU gettext code to do this. -# -# - Support Solaris .mo file formats. Unfortunately, we've been unable to -# find this format documented anywhere. import operator import os -import re import sys @@ -70,22 +66,26 @@ # https://www.gnu.org/software/gettext/manual/gettext.html#Plural-forms # http://git.savannah.gnu.org/cgit/gettext.git/tree/gettext-runtime/intl/plural.y -_token_pattern = re.compile(r""" - (?P[ \t]+) | # spaces and horizontal tabs - (?P[0-9]+\b) | # decimal integer - (?Pn\b) | # only n is allowed - (?P[()]) | - (?P[-*/%+?:]|[>, - # <=, >=, ==, !=, &&, ||, - # ? : - # unary and bitwise ops - # not allowed - (?P\w+|.) # invalid token - """, re.VERBOSE|re.DOTALL) - +_token_pattern = None def _tokenize(plural): - for mo in re.finditer(_token_pattern, plural): + global _token_pattern + if _token_pattern is None: + import re + _token_pattern = re.compile(r""" + (?P[ \t]+) | # spaces and horizontal tabs + (?P[0-9]+\b) | # decimal integer + (?Pn\b) | # only n is allowed + (?P[()]) | + (?P[-*/%+?:]|[>, + # <=, >=, ==, !=, &&, ||, + # ? : + # unary and bitwise ops + # not allowed + (?P\w+|.) # invalid token + """, re.VERBOSE|re.DOTALL) + + for mo in _token_pattern.finditer(plural): kind = mo.lastgroup if kind == 'WHITESPACES': continue @@ -648,7 +648,7 @@ def npgettext(context, msgid1, msgid2, n): # import gettext # cat = gettext.Catalog(PACKAGE, localedir=LOCALEDIR) # _ = cat.gettext -# print _('Hello World') +# print(_('Hello World')) # The resulting catalog object currently don't support access through a # dictionary API, which was supported (but apparently unused) in GNOME diff --git a/Lib/glob.py b/Lib/glob.py index c506e0e2157..f1a87c82fc5 100644 --- a/Lib/glob.py +++ b/Lib/glob.py @@ -22,6 +22,9 @@ def glob(pathname, *, root_dir=None, dir_fd=None, recursive=False, dot are special cases that are not matched by '*' and '?' patterns by default. + The order of the returned list is undefined. Sort it if you need a + particular order. + If `include_hidden` is true, the patterns '*', '?', '**' will match hidden directories. @@ -40,6 +43,9 @@ def iglob(pathname, *, root_dir=None, dir_fd=None, recursive=False, dot are special cases that are not matched by '*' and '?' patterns. + The order of the returned paths is undefined. Sort them if you need a + particular order. + If recursive is true, the pattern '**' will match any files and zero or more directories and subdirectories. """ @@ -312,24 +318,24 @@ def translate(pat, *, recursive=False, include_hidden=False, seps=None): if part: if not include_hidden and part[0] in '*?': results.append(r'(?!\.)') - results.extend(fnmatch._translate(part, f'{not_sep}*', not_sep)) + results.extend(fnmatch._translate(part, f'{not_sep}*', not_sep)[0]) if idx < last_part_idx: results.append(any_sep) res = ''.join(results) - return fr'(?s:{res})\Z' + return fr'(?s:{res})\z' @functools.lru_cache(maxsize=512) -def _compile_pattern(pat, sep, case_sensitive, recursive=True): +def _compile_pattern(pat, seps, case_sensitive, recursive=True): """Compile given glob pattern to a re.Pattern object (observing case sensitivity).""" flags = re.NOFLAG if case_sensitive else re.IGNORECASE - regex = translate(pat, recursive=recursive, include_hidden=True, seps=sep) + regex = translate(pat, recursive=recursive, include_hidden=True, seps=seps) return re.compile(regex, flags=flags).match -class _Globber: - """Class providing shell-style pattern matching and globbing. +class _GlobberBase: + """Abstract class providing shell-style pattern matching and globbing. """ def __init__(self, sep, case_sensitive, case_pedantic=False, recursive=False): @@ -338,34 +344,31 @@ def __init__(self, sep, case_sensitive, case_pedantic=False, recursive=False): self.case_pedantic = case_pedantic self.recursive = recursive - # Low-level methods - - lstat = operator.methodcaller('lstat') - add_slash = operator.methodcaller('joinpath', '') + # Abstract methods @staticmethod - def scandir(path): - """Emulates os.scandir(), which returns an object that can be used as - a context manager. This method is called by walk() and glob(). + def lexists(path): + """Implements os.path.lexists(). """ - return contextlib.nullcontext(path.iterdir()) + raise NotImplementedError @staticmethod - def concat_path(path, text): - """Appends text to the given path. + def scandir(path): + """Like os.scandir(), but generates (entry, name, path) tuples. """ - return path.with_segments(path._raw_path + text) + raise NotImplementedError @staticmethod - def parse_entry(entry): - """Returns the path of an entry yielded from scandir(). + def concat_path(path, text): + """Implements path concatenation. """ - return entry + raise NotImplementedError # High-level methods - def compile(self, pat): - return _compile_pattern(pat, self.sep, self.case_sensitive, self.recursive) + def compile(self, pat, altsep=None): + seps = (self.sep, altsep) if altsep else self.sep + return _compile_pattern(pat, seps, self.case_sensitive, self.recursive) def selector(self, parts): """Returns a function that selects from a given path, walking and @@ -387,10 +390,12 @@ def selector(self, parts): def special_selector(self, part, parts): """Returns a function that selects special children of the given path. """ + if parts: + part += self.sep select_next = self.selector(parts) def select_special(path, exists=False): - path = self.concat_path(self.add_slash(path), part) + path = self.concat_path(path, part) return select_next(path, exists) return select_special @@ -400,14 +405,16 @@ def literal_selector(self, part, parts): # Optimization: consume and join any subsequent literal parts here, # rather than leaving them for the next selector. This reduces the - # number of string concatenation operations and calls to add_slash(). + # number of string concatenation operations. while parts and magic_check.search(parts[-1]) is None: part += self.sep + parts.pop() + if parts: + part += self.sep select_next = self.selector(parts) def select_literal(path, exists=False): - path = self.concat_path(self.add_slash(path), part) + path = self.concat_path(path, part) return select_next(path, exists=False) return select_literal @@ -423,23 +430,19 @@ def wildcard_selector(self, part, parts): def select_wildcard(path, exists=False): try: - # We must close the scandir() object before proceeding to - # avoid exhausting file descriptors when globbing deep trees. - with self.scandir(path) as scandir_it: - entries = list(scandir_it) + entries = self.scandir(path) except OSError: pass else: - for entry in entries: - if match is None or match(entry.name): + for entry, entry_name, entry_path in entries: + if match is None or match(entry_name): if dir_only: try: if not entry.is_dir(): continue except OSError: continue - entry_path = self.parse_entry(entry) - if dir_only: + entry_path = self.concat_path(entry_path, self.sep) yield from select_next(entry_path, exists=True) else: yield entry_path @@ -469,7 +472,6 @@ def recursive_selector(self, part, parts): select_next = self.selector(parts) def select_recursive(path, exists=False): - path = self.add_slash(path) match_pos = len(str(path)) if match is None or match(str(path), match_pos): yield from select_next(path, exists) @@ -480,14 +482,11 @@ def select_recursive(path, exists=False): def select_recursive_step(stack, match_pos): path = stack.pop() try: - # We must close the scandir() object before proceeding to - # avoid exhausting file descriptors when globbing deep trees. - with self.scandir(path) as scandir_it: - entries = list(scandir_it) + entries = self.scandir(path) except OSError: pass else: - for entry in entries: + for entry, _entry_name, entry_path in entries: is_dir = False try: if entry.is_dir(follow_symlinks=follow_symlinks): @@ -496,8 +495,10 @@ def select_recursive_step(stack, match_pos): pass if is_dir or not dir_only: - entry_path = self.parse_entry(entry) - if match is None or match(str(entry_path), match_pos): + entry_path_str = str(entry_path) + if dir_only: + entry_path = self.concat_path(entry_path, self.sep) + if match is None or match(entry_path_str, match_pos): if dir_only: yield from select_next(entry_path, exists=True) else: @@ -516,30 +517,37 @@ def select_exists(self, path, exists=False): # Optimization: this path is already known to exist, e.g. because # it was returned from os.scandir(), so we skip calling lstat(). yield path - else: - try: - self.lstat(path) - yield path - except OSError: - pass + elif self.lexists(path): + yield path -class _StringGlobber(_Globber): - lstat = staticmethod(os.lstat) - scandir = staticmethod(os.scandir) - parse_entry = operator.attrgetter('path') +class _StringGlobber(_GlobberBase): + """Provides shell-style pattern matching and globbing for string paths. + """ + lexists = staticmethod(os.path.lexists) concat_path = operator.add - if os.name == 'nt': - @staticmethod - def add_slash(pathname): - tail = os.path.splitroot(pathname)[2] - if not tail or tail[-1] in '\\/': - return pathname - return f'{pathname}\\' - else: - @staticmethod - def add_slash(pathname): - if not pathname or pathname[-1] == '/': - return pathname - return f'{pathname}/' + @staticmethod + def scandir(path): + # We must close the scandir() object before proceeding to + # avoid exhausting file descriptors when globbing deep trees. + with os.scandir(path) as scandir_it: + entries = list(scandir_it) + return ((entry, entry.name, entry.path) for entry in entries) + + +class _PathGlobber(_GlobberBase): + """Provides shell-style pattern matching and globbing for pathlib paths. + """ + + @staticmethod + def lexists(path): + return path.info.exists(follow_symlinks=False) + + @staticmethod + def scandir(path): + return ((child.info, child.name, child) for child in path.iterdir()) + + @staticmethod + def concat_path(path, text): + return path.with_segments(str(path) + text) diff --git a/Lib/hashlib.py b/Lib/hashlib.py index e5f81d754ac..0e9bd98aa1f 100644 --- a/Lib/hashlib.py +++ b/Lib/hashlib.py @@ -33,7 +33,7 @@ - hexdigest(): Like digest() except the digest is returned as a string of double length, containing only hexadecimal digits. - copy(): Return a copy (clone) of the hash object. This can be used to - efficiently compute the digests of datas that share a common + efficiently compute the digests of data that share a common initial substring. For example, to obtain the digest of the byte string 'Nobody inspects the @@ -65,7 +65,7 @@ algorithms_available = set(__always_supported) __all__ = __always_supported + ('new', 'algorithms_guaranteed', - 'algorithms_available', 'pbkdf2_hmac', 'file_digest') + 'algorithms_available', 'file_digest') __builtin_constructor_cache = {} @@ -92,13 +92,13 @@ def __get_builtin_constructor(name): import _md5 cache['MD5'] = cache['md5'] = _md5.md5 elif name in {'SHA256', 'sha256', 'SHA224', 'sha224'}: - import _sha256 - cache['SHA224'] = cache['sha224'] = _sha256.sha224 - cache['SHA256'] = cache['sha256'] = _sha256.sha256 + import _sha2 + cache['SHA224'] = cache['sha224'] = _sha2.sha224 + cache['SHA256'] = cache['sha256'] = _sha2.sha256 elif name in {'SHA512', 'sha512', 'SHA384', 'sha384'}: - import _sha512 - cache['SHA384'] = cache['sha384'] = _sha512.sha384 - cache['SHA512'] = cache['sha512'] = _sha512.sha512 + import _sha2 + cache['SHA384'] = cache['sha384'] = _sha2.sha384 + cache['SHA512'] = cache['sha512'] = _sha2.sha512 elif name in {'blake2b', 'blake2s'}: import _blake2 cache['blake2b'] = _blake2.blake2b @@ -141,38 +141,37 @@ def __get_openssl_constructor(name): return __get_builtin_constructor(name) -def __py_new(name, data=b'', **kwargs): +def __py_new(name, *args, **kwargs): """new(name, data=b'', **kwargs) - Return a new hashing object using the named algorithm; optionally initialized with data (which must be a bytes-like object). """ - return __get_builtin_constructor(name)(data, **kwargs) + return __get_builtin_constructor(name)(*args, **kwargs) -def __hash_new(name, data=b'', **kwargs): +def __hash_new(name, *args, **kwargs): """new(name, data=b'') - Return a new hashing object using the named algorithm; optionally initialized with data (which must be a bytes-like object). """ if name in __block_openssl_constructor: # Prefer our builtin blake2 implementation. - return __get_builtin_constructor(name)(data, **kwargs) + return __get_builtin_constructor(name)(*args, **kwargs) try: - return _hashlib.new(name, data, **kwargs) + return _hashlib.new(name, *args, **kwargs) except ValueError: # If the _hashlib module (OpenSSL) doesn't support the named # hash, try using our builtin implementations. # This allows for SHA224/256 and SHA384/512 support even though # the OpenSSL library prior to 0.9.8 doesn't provide them. - return __get_builtin_constructor(name)(data) + return __get_builtin_constructor(name)(*args, **kwargs) try: import _hashlib new = __hash_new __get_hash = __get_openssl_constructor - # TODO: RUSTPYTHON set in _hashlib instance PyFrozenSet algorithms_available - '''algorithms_available = algorithms_available.union( - _hashlib.openssl_md_meth_names)''' + algorithms_available = algorithms_available.union( + _hashlib.openssl_md_meth_names) except ImportError: _hashlib = None new = __py_new @@ -181,76 +180,14 @@ def __hash_new(name, data=b'', **kwargs): try: # OpenSSL's PKCS5_PBKDF2_HMAC requires OpenSSL 1.0+ with HMAC and SHA from _hashlib import pbkdf2_hmac + __all__ += ('pbkdf2_hmac',) except ImportError: - from warnings import warn as _warn - _trans_5C = bytes((x ^ 0x5C) for x in range(256)) - _trans_36 = bytes((x ^ 0x36) for x in range(256)) - - def pbkdf2_hmac(hash_name, password, salt, iterations, dklen=None): - """Password based key derivation function 2 (PKCS #5 v2.0) - - This Python implementations based on the hmac module about as fast - as OpenSSL's PKCS5_PBKDF2_HMAC for short passwords and much faster - for long passwords. - """ - _warn( - "Python implementation of pbkdf2_hmac() is deprecated.", - category=DeprecationWarning, - stacklevel=2 - ) - if not isinstance(hash_name, str): - raise TypeError(hash_name) - - if not isinstance(password, (bytes, bytearray)): - password = bytes(memoryview(password)) - if not isinstance(salt, (bytes, bytearray)): - salt = bytes(memoryview(salt)) - - # Fast inline HMAC implementation - inner = new(hash_name) - outer = new(hash_name) - blocksize = getattr(inner, 'block_size', 64) - if len(password) > blocksize: - password = new(hash_name, password).digest() - password = password + b'\x00' * (blocksize - len(password)) - inner.update(password.translate(_trans_36)) - outer.update(password.translate(_trans_5C)) - - def prf(msg, inner=inner, outer=outer): - # PBKDF2_HMAC uses the password as key. We can re-use the same - # digest objects and just update copies to skip initialization. - icpy = inner.copy() - ocpy = outer.copy() - icpy.update(msg) - ocpy.update(icpy.digest()) - return ocpy.digest() - - if iterations < 1: - raise ValueError(iterations) - if dklen is None: - dklen = outer.digest_size - if dklen < 1: - raise ValueError(dklen) - - dkey = b'' - loop = 1 - from_bytes = int.from_bytes - while len(dkey) < dklen: - prev = prf(salt + loop.to_bytes(4)) - # endianness doesn't matter here as long to / from use the same - rkey = from_bytes(prev) - for i in range(iterations - 1): - prev = prf(prev) - # rkey = rkey ^ prev - rkey ^= from_bytes(prev) - loop += 1 - dkey += rkey.to_bytes(inner.digest_size) - - return dkey[:dklen] + pass + try: # OpenSSL's scrypt requires OpenSSL 1.1+ - from _hashlib import scrypt + from _hashlib import scrypt # noqa: F401 except ImportError: pass @@ -294,6 +231,8 @@ def file_digest(fileobj, digest, /, *, _bufsize=2**18): view = memoryview(buf) while True: size = fileobj.readinto(buf) + if size is None: + raise BlockingIOError("I/O operation would block.") if size == 0: break # EOF digestobj.update(view[:size]) diff --git a/Lib/hmac.py b/Lib/hmac.py index 8b4eb2fe741..2d6016cda11 100644 --- a/Lib/hmac.py +++ b/Lib/hmac.py @@ -3,7 +3,6 @@ Implements the HMAC algorithm as described by RFC 2104. """ -import warnings as _warnings try: import _hashlib as _hashopenssl except ImportError: @@ -14,7 +13,10 @@ compare_digest = _hashopenssl.compare_digest _functype = type(_hashopenssl.openssl_sha256) # builtin type -import hashlib as _hashlib +try: + import _hmac +except ImportError: + _hmac = None trans_5C = bytes((x ^ 0x5C) for x in range(256)) trans_36 = bytes((x ^ 0x36) for x in range(256)) @@ -24,11 +26,27 @@ digest_size = None +def _get_digest_constructor(digest_like): + if callable(digest_like): + return digest_like + if isinstance(digest_like, str): + def digest_wrapper(d=b''): + import hashlib + return hashlib.new(digest_like, d) + else: + def digest_wrapper(d=b''): + return digest_like.new(d) + return digest_wrapper + + class HMAC: """RFC 2104 HMAC class. Also complies with RFC 4231. This supports the API for Cryptographic Hash Functions (PEP 247). """ + + # Note: self.blocksize is the default blocksize; self.block_size + # is effective block size as well as the public API attribute. blocksize = 64 # 512-bit HMAC; can be changed in subclasses. __slots__ = ( @@ -50,31 +68,47 @@ def __init__(self, key, msg=None, digestmod=''): """ if not isinstance(key, (bytes, bytearray)): - raise TypeError("key: expected bytes or bytearray, but got %r" % type(key).__name__) + raise TypeError(f"key: expected bytes or bytearray, " + f"but got {type(key).__name__!r}") if not digestmod: raise TypeError("Missing required argument 'digestmod'.") + self.__init(key, msg, digestmod) + + def __init(self, key, msg, digestmod): if _hashopenssl and isinstance(digestmod, (str, _functype)): try: - self._init_hmac(key, msg, digestmod) - except _hashopenssl.UnsupportedDigestmodError: - self._init_old(key, msg, digestmod) - else: - self._init_old(key, msg, digestmod) + self._init_openssl_hmac(key, msg, digestmod) + return + except _hashopenssl.UnsupportedDigestmodError: # pragma: no cover + pass + if _hmac and isinstance(digestmod, str): + try: + self._init_builtin_hmac(key, msg, digestmod) + return + except _hmac.UnknownHashError: # pragma: no cover + pass + self._init_old(key, msg, digestmod) - def _init_hmac(self, key, msg, digestmod): + def _init_openssl_hmac(self, key, msg, digestmod): self._hmac = _hashopenssl.hmac_new(key, msg, digestmod=digestmod) + self._inner = self._outer = None # because the slots are defined + self.digest_size = self._hmac.digest_size + self.block_size = self._hmac.block_size + + _init_hmac = _init_openssl_hmac # for backward compatibility (if any) + + def _init_builtin_hmac(self, key, msg, digestmod): + self._hmac = _hmac.new(key, msg, digestmod=digestmod) + self._inner = self._outer = None # because the slots are defined self.digest_size = self._hmac.digest_size self.block_size = self._hmac.block_size def _init_old(self, key, msg, digestmod): - if callable(digestmod): - digest_cons = digestmod - elif isinstance(digestmod, str): - digest_cons = lambda d=b'': _hashlib.new(digestmod, d) - else: - digest_cons = lambda d=b'': digestmod.new(d) + import warnings + + digest_cons = _get_digest_constructor(digestmod) self._hmac = None self._outer = digest_cons() @@ -84,21 +118,19 @@ def _init_old(self, key, msg, digestmod): if hasattr(self._inner, 'block_size'): blocksize = self._inner.block_size if blocksize < 16: - _warnings.warn('block_size of %d seems too small; using our ' - 'default of %d.' % (blocksize, self.blocksize), - RuntimeWarning, 2) - blocksize = self.blocksize + warnings.warn(f"block_size of {blocksize} seems too small; " + f"using our default of {self.blocksize}.", + RuntimeWarning, 2) + blocksize = self.blocksize # pragma: no cover else: - _warnings.warn('No block_size attribute on given digest object; ' - 'Assuming %d.' % (self.blocksize), - RuntimeWarning, 2) - blocksize = self.blocksize + warnings.warn("No block_size attribute on given digest object; " + f"Assuming {self.blocksize}.", + RuntimeWarning, 2) + blocksize = self.blocksize # pragma: no cover if len(key) > blocksize: key = digest_cons(key).digest() - # self.blocksize is the default blocksize. self.block_size is - # effective block size as well as the public API attribute. self.block_size = blocksize key = key.ljust(blocksize, b'\0') @@ -127,6 +159,7 @@ def copy(self): # Call __new__ directly to avoid the expensive __init__. other = self.__class__.__new__(self.__class__) other.digest_size = self.digest_size + other.block_size = self.block_size if self._hmac: other._hmac = self._hmac.copy() other._inner = other._outer = None @@ -164,6 +197,7 @@ def hexdigest(self): h = self._current() return h.hexdigest() + def new(key, msg=None, digestmod=''): """Create a new hashing object and return it. @@ -193,25 +227,41 @@ def digest(key, msg, digest): A hashlib constructor returning a new hash object. *OR* A module supporting PEP 247. """ - if _hashopenssl is not None and isinstance(digest, (str, _functype)): + if _hashopenssl and isinstance(digest, (str, _functype)): try: return _hashopenssl.hmac_digest(key, msg, digest) + except OverflowError: + # OpenSSL's HMAC limits the size of the key to INT_MAX. + # Instead of falling back to HACL* implementation which + # may still not be supported due to a too large key, we + # directly switch to the pure Python fallback instead + # even if we could have used streaming HMAC for small keys + # but large messages. + return _compute_digest_fallback(key, msg, digest) except _hashopenssl.UnsupportedDigestmodError: pass - if callable(digest): - digest_cons = digest - elif isinstance(digest, str): - digest_cons = lambda d=b'': _hashlib.new(digest, d) - else: - digest_cons = lambda d=b'': digest.new(d) + if _hmac and isinstance(digest, str): + try: + return _hmac.compute_digest(key, msg, digest) + except (OverflowError, _hmac.UnknownHashError): + # HACL* HMAC limits the size of the key to UINT32_MAX + # so we fallback to the pure Python implementation even + # if streaming HMAC may have been used for small keys + # and large messages. + pass + + return _compute_digest_fallback(key, msg, digest) + +def _compute_digest_fallback(key, msg, digest): + digest_cons = _get_digest_constructor(digest) inner = digest_cons() outer = digest_cons() blocksize = getattr(inner, 'block_size', 64) if len(key) > blocksize: key = digest_cons(key).digest() - key = key + b'\x00' * (blocksize - len(key)) + key = key.ljust(blocksize, b'\0') inner.update(key.translate(trans_36)) outer.update(key.translate(trans_5C)) inner.update(msg) diff --git a/Lib/http/__init__.py b/Lib/http/__init__.py index 17a47b180e5..691b4a9a367 100644 --- a/Lib/http/__init__.py +++ b/Lib/http/__init__.py @@ -54,8 +54,9 @@ def is_server_error(self): CONTINUE = 100, 'Continue', 'Request received, please continue' SWITCHING_PROTOCOLS = (101, 'Switching Protocols', 'Switching to new protocol; obey Upgrade header') - PROCESSING = 102, 'Processing' - EARLY_HINTS = 103, 'Early Hints' + PROCESSING = 102, 'Processing', 'Server is processing the request' + EARLY_HINTS = (103, 'Early Hints', + 'Headers sent to prepare for the response') # success OK = 200, 'OK', 'Request fulfilled, document follows' @@ -67,9 +68,11 @@ def is_server_error(self): NO_CONTENT = 204, 'No Content', 'Request fulfilled, nothing follows' RESET_CONTENT = 205, 'Reset Content', 'Clear input form for further input' PARTIAL_CONTENT = 206, 'Partial Content', 'Partial content follows' - MULTI_STATUS = 207, 'Multi-Status' - ALREADY_REPORTED = 208, 'Already Reported' - IM_USED = 226, 'IM Used' + MULTI_STATUS = (207, 'Multi-Status', + 'Response contains multiple statuses in the body') + ALREADY_REPORTED = (208, 'Already Reported', + 'Operation has already been reported') + IM_USED = 226, 'IM Used', 'Request completed using instance manipulations' # redirection MULTIPLE_CHOICES = (300, 'Multiple Choices', @@ -128,15 +131,19 @@ def is_server_error(self): EXPECTATION_FAILED = (417, 'Expectation Failed', 'Expect condition could not be satisfied') IM_A_TEAPOT = (418, 'I\'m a Teapot', - 'Server refuses to brew coffee because it is a teapot.') + 'Server refuses to brew coffee because it is a teapot') MISDIRECTED_REQUEST = (421, 'Misdirected Request', 'Server is not able to produce a response') - UNPROCESSABLE_CONTENT = 422, 'Unprocessable Content' + UNPROCESSABLE_CONTENT = (422, 'Unprocessable Content', + 'Server is not able to process the contained instructions') UNPROCESSABLE_ENTITY = UNPROCESSABLE_CONTENT - LOCKED = 423, 'Locked' - FAILED_DEPENDENCY = 424, 'Failed Dependency' - TOO_EARLY = 425, 'Too Early' - UPGRADE_REQUIRED = 426, 'Upgrade Required' + LOCKED = 423, 'Locked', 'Resource of a method is locked' + FAILED_DEPENDENCY = (424, 'Failed Dependency', + 'Dependent action of the request failed') + TOO_EARLY = (425, 'Too Early', + 'Server refuses to process a request that might be replayed') + UPGRADE_REQUIRED = (426, 'Upgrade Required', + 'Server refuses to perform the request using the current protocol') PRECONDITION_REQUIRED = (428, 'Precondition Required', 'The origin server requires the request to be conditional') TOO_MANY_REQUESTS = (429, 'Too Many Requests', @@ -164,10 +171,14 @@ def is_server_error(self): 'The gateway server did not receive a timely response') HTTP_VERSION_NOT_SUPPORTED = (505, 'HTTP Version Not Supported', 'Cannot fulfill request') - VARIANT_ALSO_NEGOTIATES = 506, 'Variant Also Negotiates' - INSUFFICIENT_STORAGE = 507, 'Insufficient Storage' - LOOP_DETECTED = 508, 'Loop Detected' - NOT_EXTENDED = 510, 'Not Extended' + VARIANT_ALSO_NEGOTIATES = (506, 'Variant Also Negotiates', + 'Server has an internal configuration error') + INSUFFICIENT_STORAGE = (507, 'Insufficient Storage', + 'Server is not able to store the representation') + LOOP_DETECTED = (508, 'Loop Detected', + 'Server encountered an infinite loop while processing a request') + NOT_EXTENDED = (510, 'Not Extended', + 'Request does not meet the resource access policy') NETWORK_AUTHENTICATION_REQUIRED = (511, 'Network Authentication Required', 'The client needs to authenticate to gain network access') diff --git a/Lib/http/client.py b/Lib/http/client.py index dd5f4136e9e..77f8d26291d 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py @@ -1047,7 +1047,7 @@ def close(self): response.close() def send(self, data): - """Send `data' to the server. + """Send 'data' to the server. ``data`` can be a string object, a bytes object, an array object, a file-like object that supports a .read() method, or an iterable object. """ @@ -1159,10 +1159,10 @@ def putrequest(self, method, url, skip_host=False, skip_accept_encoding=False): """Send a request to the server. - `method' specifies an HTTP request method, e.g. 'GET'. - `url' specifies the object being requested, e.g. '/index.html'. - `skip_host' if True does not add automatically a 'Host:' header - `skip_accept_encoding' if True does not add automatically an + 'method' specifies an HTTP request method, e.g. 'GET'. + 'url' specifies the object being requested, e.g. '/index.html'. + 'skip_host' if True does not add automatically a 'Host:' header + 'skip_accept_encoding' if True does not add automatically an 'Accept-Encoding:' header """ diff --git a/Lib/http/cookiejar.py b/Lib/http/cookiejar.py index 9a2f0fb851c..68cf16c93cc 100644 --- a/Lib/http/cookiejar.py +++ b/Lib/http/cookiejar.py @@ -1987,7 +1987,7 @@ class MozillaCookieJar(FileCookieJar): This class differs from CookieJar only in the format it uses to save and load cookies to and from a file. This class uses the Mozilla/Netscape - `cookies.txt' format. curl and lynx use this file format, too. + 'cookies.txt' format. curl and lynx use this file format, too. Don't expect cookies saved while the browser is running to be noticed by the browser (in fact, Mozilla on unix will overwrite your saved cookies if diff --git a/Lib/http/cookies.py b/Lib/http/cookies.py index 57791c6ab08..e0e2cd4b696 100644 --- a/Lib/http/cookies.py +++ b/Lib/http/cookies.py @@ -87,9 +87,9 @@ such trickeries do not confuse it. >>> C = cookies.SimpleCookie() - >>> C.load('keebler="E=everybody; L=\\"Loves\\"; fudge=\\012;";') + >>> C.load('keebler="E=everybody; L=\\"Loves\\"; fudge=;";') >>> print(C) - Set-Cookie: keebler="E=everybody; L=\"Loves\"; fudge=\012;" + Set-Cookie: keebler="E=everybody; L=\"Loves\"; fudge=;" Each element of the Cookie also supports all of the RFC 2109 Cookie attributes. Here's an example which sets the Path @@ -170,6 +170,15 @@ class CookieError(Exception): }) _is_legal_key = re.compile('[%s]+' % re.escape(_LegalChars)).fullmatch +_control_character_re = re.compile(r'[\x00-\x1F\x7F]') + + +def _has_control_character(*val): + """Detects control characters within a value. + Supports any type, as header values can be any type. + """ + return any(_control_character_re.search(str(v)) for v in val) + def _quote(str): r"""Quote a string for use in a cookie header. @@ -264,17 +273,19 @@ class Morsel(dict): "httponly" : "HttpOnly", "version" : "Version", "samesite" : "SameSite", + "partitioned": "Partitioned", } - _flags = {'secure', 'httponly'} + _reserved_defaults = dict.fromkeys(_reserved, "") + + _flags = {'secure', 'httponly', 'partitioned'} def __init__(self): # Set defaults self._key = self._value = self._coded_value = None # Set default attributes - for key in self._reserved: - dict.__setitem__(self, key, "") + dict.update(self, self._reserved_defaults) @property def key(self): @@ -292,12 +303,16 @@ def __setitem__(self, K, V): K = K.lower() if not K in self._reserved: raise CookieError("Invalid attribute %r" % (K,)) + if _has_control_character(K, V): + raise CookieError(f"Control characters are not allowed in cookies {K!r} {V!r}") dict.__setitem__(self, K, V) def setdefault(self, key, val=None): key = key.lower() if key not in self._reserved: raise CookieError("Invalid attribute %r" % (key,)) + if _has_control_character(key, val): + raise CookieError("Control characters are not allowed in cookies %r %r" % (key, val,)) return dict.setdefault(self, key, val) def __eq__(self, morsel): @@ -333,6 +348,9 @@ def set(self, key, val, coded_val): raise CookieError('Attempt to set a reserved key %r' % (key,)) if not _is_legal_key(key): raise CookieError('Illegal key %r' % (key,)) + if _has_control_character(key, val, coded_val): + raise CookieError( + "Control characters are not allowed in cookies %r %r %r" % (key, val, coded_val,)) # It's a good key, so save it. self._key = key @@ -486,7 +504,10 @@ def output(self, attrs=None, header="Set-Cookie:", sep="\015\012"): result = [] items = sorted(self.items()) for key, value in items: - result.append(value.output(attrs, header)) + value_output = value.output(attrs, header) + if _has_control_character(value_output): + raise CookieError("Control characters are not allowed in cookies") + result.append(value_output) return sep.join(result) __str__ = output diff --git a/Lib/http/server.py b/Lib/http/server.py index 0ec479003a4..ac1f57c29f0 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -83,8 +83,10 @@ __version__ = "0.6" __all__ = [ - "HTTPServer", "ThreadingHTTPServer", "BaseHTTPRequestHandler", - "SimpleHTTPRequestHandler", "CGIHTTPRequestHandler", + "HTTPServer", "ThreadingHTTPServer", + "HTTPSServer", "ThreadingHTTPSServer", + "BaseHTTPRequestHandler", "SimpleHTTPRequestHandler", + "CGIHTTPRequestHandler", ] import copy @@ -99,7 +101,7 @@ import posixpath import select import shutil -import socket # For gethostbyaddr() +import socket import socketserver import sys import time @@ -114,6 +116,11 @@ + Error response @@ -133,7 +140,8 @@ class HTTPServer(socketserver.TCPServer): - allow_reuse_address = 1 # Seems to make sense in testing environment + allow_reuse_address = True # Seems to make sense in testing environment + allow_reuse_port = False def server_bind(self): """Override server_bind to store the server name.""" @@ -147,6 +155,47 @@ class ThreadingHTTPServer(socketserver.ThreadingMixIn, HTTPServer): daemon_threads = True +class HTTPSServer(HTTPServer): + def __init__(self, server_address, RequestHandlerClass, + bind_and_activate=True, *, certfile, keyfile=None, + password=None, alpn_protocols=None): + try: + import ssl + except ImportError: + raise RuntimeError("SSL module is missing; " + "HTTPS support is unavailable") + + self.ssl = ssl + self.certfile = certfile + self.keyfile = keyfile + self.password = password + # Support by default HTTP/1.1 + self.alpn_protocols = ( + ["http/1.1"] if alpn_protocols is None else alpn_protocols + ) + + super().__init__(server_address, + RequestHandlerClass, + bind_and_activate) + + def server_activate(self): + """Wrap the socket in SSLSocket.""" + super().server_activate() + context = self._create_context() + self.socket = context.wrap_socket(self.socket, server_side=True) + + def _create_context(self): + """Create a secure SSL context.""" + context = self.ssl.create_default_context(self.ssl.Purpose.CLIENT_AUTH) + context.load_cert_chain(self.certfile, self.keyfile, self.password) + context.set_alpn_protocols(self.alpn_protocols) + return context + + +class ThreadingHTTPSServer(socketserver.ThreadingMixIn, HTTPSServer): + daemon_threads = True + + class BaseHTTPRequestHandler(socketserver.StreamRequestHandler): """HTTP request handler base class. @@ -817,6 +866,7 @@ def list_directory(self, path): r.append('') r.append('') r.append(f'') + r.append('') r.append(f'{title}\n') r.append(f'\n

{title}

') r.append('
\n
    ') @@ -1281,7 +1331,8 @@ def _get_best_family(*address): def test(HandlerClass=BaseHTTPRequestHandler, ServerClass=ThreadingHTTPServer, - protocol="HTTP/1.0", port=8000, bind=None): + protocol="HTTP/1.0", port=8000, bind=None, + tls_cert=None, tls_key=None, tls_password=None): """Test the HTTP request handler class. This runs an HTTP server on port 8000 (or the port argument). @@ -1289,12 +1340,20 @@ def test(HandlerClass=BaseHTTPRequestHandler, """ ServerClass.address_family, addr = _get_best_family(bind, port) HandlerClass.protocol_version = protocol - with ServerClass(addr, HandlerClass) as httpd: + + if tls_cert: + server = ServerClass(addr, HandlerClass, certfile=tls_cert, + keyfile=tls_key, password=tls_password) + else: + server = ServerClass(addr, HandlerClass) + + with server as httpd: host, port = httpd.socket.getsockname()[:2] url_host = f'[{host}]' if ':' in host else host + protocol = 'HTTPS' if tls_cert else 'HTTP' print( - f"Serving HTTP on {host} port {port} " - f"(http://{url_host}:{port}/) ..." + f"Serving {protocol} on {host} port {port} " + f"({protocol.lower()}://{url_host}:{port}/) ..." ) try: httpd.serve_forever() @@ -1306,7 +1365,7 @@ def test(HandlerClass=BaseHTTPRequestHandler, import argparse import contextlib - parser = argparse.ArgumentParser() + parser = argparse.ArgumentParser(color=True) parser.add_argument('--cgi', action='store_true', help='run as CGI server') parser.add_argument('-b', '--bind', metavar='ADDRESS', @@ -1319,17 +1378,38 @@ def test(HandlerClass=BaseHTTPRequestHandler, default='HTTP/1.0', help='conform to this HTTP version ' '(default: %(default)s)') + parser.add_argument('--tls-cert', metavar='PATH', + help='path to the TLS certificate chain file') + parser.add_argument('--tls-key', metavar='PATH', + help='path to the TLS key file') + parser.add_argument('--tls-password-file', metavar='PATH', + help='path to the password file for the TLS key') parser.add_argument('port', default=8000, type=int, nargs='?', help='bind to this port ' '(default: %(default)s)') args = parser.parse_args() + + if not args.tls_cert and args.tls_key: + parser.error("--tls-key requires --tls-cert to be set") + + tls_key_password = None + if args.tls_password_file: + if not args.tls_cert: + parser.error("--tls-password-file requires --tls-cert to be set") + + try: + with open(args.tls_password_file, "r", encoding="utf-8") as f: + tls_key_password = f.read().strip() + except OSError as e: + parser.error(f"Failed to read TLS password file: {e}") + if args.cgi: handler_class = CGIHTTPRequestHandler else: handler_class = SimpleHTTPRequestHandler # ensure dual-stack is not disabled; ref #38907 - class DualStackServer(ThreadingHTTPServer): + class DualStackServerMixin: def server_bind(self): # suppress exception when protocol is IPv4 @@ -1342,10 +1422,20 @@ def finish_request(self, request, client_address): self.RequestHandlerClass(request, client_address, self, directory=args.directory) + class HTTPDualStackServer(DualStackServerMixin, ThreadingHTTPServer): + pass + class HTTPSDualStackServer(DualStackServerMixin, ThreadingHTTPSServer): + pass + + ServerClass = HTTPSDualStackServer if args.tls_cert else HTTPDualStackServer + test( HandlerClass=handler_class, - ServerClass=DualStackServer, + ServerClass=ServerClass, port=args.port, bind=args.bind, protocol=args.protocol, + tls_cert=args.tls_cert, + tls_key=args.tls_key, + tls_password=tls_key_password, ) diff --git a/Lib/imaplib.py b/Lib/imaplib.py new file mode 100644 index 00000000000..cbe129b3e7c --- /dev/null +++ b/Lib/imaplib.py @@ -0,0 +1,1967 @@ +"""IMAP4 client. + +Based on RFC 2060. + +Public class: IMAP4 +Public variable: Debug +Public functions: Internaldate2tuple + Int2AP + ParseFlags + Time2Internaldate +""" + +# Author: Piers Lauder December 1997. +# +# Authentication code contributed by Donn Cave June 1998. +# String method conversion by ESR, February 2001. +# GET/SETACL contributed by Anthony Baxter April 2001. +# IMAP4_SSL contributed by Tino Lange March 2002. +# GET/SETQUOTA contributed by Andreas Zeidler June 2002. +# PROXYAUTH contributed by Rick Holbert November 2002. +# GET/SETANNOTATION contributed by Tomas Lindroos June 2005. +# IDLE contributed by Forest August 2024. + +__version__ = "2.60" + +import binascii, errno, random, re, socket, subprocess, sys, time, calendar +from datetime import datetime, timezone, timedelta +from io import DEFAULT_BUFFER_SIZE + +try: + import ssl + HAVE_SSL = True +except ImportError: + HAVE_SSL = False + +__all__ = ["IMAP4", "IMAP4_stream", "Internaldate2tuple", + "Int2AP", "ParseFlags", "Time2Internaldate"] + +# Globals + +CRLF = b'\r\n' +Debug = 0 +IMAP4_PORT = 143 +IMAP4_SSL_PORT = 993 +AllowedVersions = ('IMAP4REV1', 'IMAP4') # Most recent first + +# Maximal line length when calling readline(). This is to prevent +# reading arbitrary length lines. RFC 3501 and 2060 (IMAP 4rev1) +# don't specify a line length. RFC 2683 suggests limiting client +# command lines to 1000 octets and that servers should be prepared +# to accept command lines up to 8000 octets, so we used to use 10K here. +# In the modern world (eg: gmail) the response to, for example, a +# search command can be quite large, so we now use 1M. +_MAXLINE = 1000000 + + +# Commands + +Commands = { + # name valid states + 'APPEND': ('AUTH', 'SELECTED'), + 'AUTHENTICATE': ('NONAUTH',), + 'CAPABILITY': ('NONAUTH', 'AUTH', 'SELECTED', 'LOGOUT'), + 'CHECK': ('SELECTED',), + 'CLOSE': ('SELECTED',), + 'COPY': ('SELECTED',), + 'CREATE': ('AUTH', 'SELECTED'), + 'DELETE': ('AUTH', 'SELECTED'), + 'DELETEACL': ('AUTH', 'SELECTED'), + 'ENABLE': ('AUTH', ), + 'EXAMINE': ('AUTH', 'SELECTED'), + 'EXPUNGE': ('SELECTED',), + 'FETCH': ('SELECTED',), + 'GETACL': ('AUTH', 'SELECTED'), + 'GETANNOTATION':('AUTH', 'SELECTED'), + 'GETQUOTA': ('AUTH', 'SELECTED'), + 'GETQUOTAROOT': ('AUTH', 'SELECTED'), + 'IDLE': ('AUTH', 'SELECTED'), + 'MYRIGHTS': ('AUTH', 'SELECTED'), + 'LIST': ('AUTH', 'SELECTED'), + 'LOGIN': ('NONAUTH',), + 'LOGOUT': ('NONAUTH', 'AUTH', 'SELECTED', 'LOGOUT'), + 'LSUB': ('AUTH', 'SELECTED'), + 'MOVE': ('SELECTED',), + 'NAMESPACE': ('AUTH', 'SELECTED'), + 'NOOP': ('NONAUTH', 'AUTH', 'SELECTED', 'LOGOUT'), + 'PARTIAL': ('SELECTED',), # NB: obsolete + 'PROXYAUTH': ('AUTH',), + 'RENAME': ('AUTH', 'SELECTED'), + 'SEARCH': ('SELECTED',), + 'SELECT': ('AUTH', 'SELECTED'), + 'SETACL': ('AUTH', 'SELECTED'), + 'SETANNOTATION':('AUTH', 'SELECTED'), + 'SETQUOTA': ('AUTH', 'SELECTED'), + 'SORT': ('SELECTED',), + 'STARTTLS': ('NONAUTH',), + 'STATUS': ('AUTH', 'SELECTED'), + 'STORE': ('SELECTED',), + 'SUBSCRIBE': ('AUTH', 'SELECTED'), + 'THREAD': ('SELECTED',), + 'UID': ('SELECTED',), + 'UNSUBSCRIBE': ('AUTH', 'SELECTED'), + 'UNSELECT': ('SELECTED',), + } + +# Patterns to match server responses + +Continuation = re.compile(br'\+( (?P.*))?') +Flags = re.compile(br'.*FLAGS \((?P[^\)]*)\)') +InternalDate = re.compile(br'.*INTERNALDATE "' + br'(?P[ 0123][0-9])-(?P[A-Z][a-z][a-z])-(?P[0-9][0-9][0-9][0-9])' + br' (?P[0-9][0-9]):(?P[0-9][0-9]):(?P[0-9][0-9])' + br' (?P[-+])(?P[0-9][0-9])(?P[0-9][0-9])' + br'"') +# Literal is no longer used; kept for backward compatibility. +Literal = re.compile(br'.*{(?P\d+)}$', re.ASCII) +MapCRLF = re.compile(br'\r\n|\r|\n') +# We no longer exclude the ']' character from the data portion of the response +# code, even though it violates the RFC. Popular IMAP servers such as Gmail +# allow flags with ']', and there are programs (including imaplib!) that can +# produce them. The problem with this is if the 'text' portion of the response +# includes a ']' we'll parse the response wrong (which is the point of the RFC +# restriction). However, that seems less likely to be a problem in practice +# than being unable to correctly parse flags that include ']' chars, which +# was reported as a real-world problem in issue #21815. +Response_code = re.compile(br'\[(?P[A-Z-]+)( (?P.*))?\]') +Untagged_response = re.compile(br'\* (?P[A-Z-]+)( (?P.*))?') +# Untagged_status is no longer used; kept for backward compatibility +Untagged_status = re.compile( + br'\* (?P\d+) (?P[A-Z-]+)( (?P.*))?', re.ASCII) +# We compile these in _mode_xxx. +_Literal = br'.*{(?P\d+)}$' +_Untagged_status = br'\* (?P\d+) (?P[A-Z-]+)( (?P.*))?' + + + +class IMAP4: + + r"""IMAP4 client class. + + Instantiate with: IMAP4([host[, port[, timeout=None]]]) + + host - host's name (default: localhost); + port - port number (default: standard IMAP4 port). + timeout - socket timeout (default: None) + If timeout is not given or is None, + the global default socket timeout is used + + All IMAP4rev1 commands are supported by methods of the same + name (in lowercase). + + All arguments to commands are converted to strings, except for + AUTHENTICATE, and the last argument to APPEND which is passed as + an IMAP4 literal. If necessary (the string contains any + non-printing characters or white-space and isn't enclosed with + either parentheses or double quotes) each string is quoted. + However, the 'password' argument to the LOGIN command is always + quoted. If you want to avoid having an argument string quoted + (eg: the 'flags' argument to STORE) then enclose the string in + parentheses (eg: "(\Deleted)"). + + Each command returns a tuple: (type, [data, ...]) where 'type' + is usually 'OK' or 'NO', and 'data' is either the text from the + tagged response, or untagged results from command. Each 'data' + is either a string, or a tuple. If a tuple, then the first part + is the header of the response, and the second part contains + the data (ie: 'literal' value). + + Errors raise the exception class .error(""). + IMAP4 server errors raise .abort(""), + which is a sub-class of 'error'. Mailbox status changes + from READ-WRITE to READ-ONLY raise the exception class + .readonly(""), which is a sub-class of 'abort'. + + "error" exceptions imply a program error. + "abort" exceptions imply the connection should be reset, and + the command re-tried. + "readonly" exceptions imply the command should be re-tried. + + Note: to use this module, you must read the RFCs pertaining to the + IMAP4 protocol, as the semantics of the arguments to each IMAP4 + command are left to the invoker, not to mention the results. Also, + most IMAP servers implement a sub-set of the commands available here. + """ + + class error(Exception): pass # Logical errors - debug required + class abort(error): pass # Service errors - close and retry + class readonly(abort): pass # Mailbox status changed to READ-ONLY + class _responsetimeout(TimeoutError): pass # No response during IDLE + + def __init__(self, host='', port=IMAP4_PORT, timeout=None): + self.debug = Debug + self.state = 'LOGOUT' + self.literal = None # A literal argument to a command + self.tagged_commands = {} # Tagged commands awaiting response + self.untagged_responses = {} # {typ: [data, ...], ...} + self.continuation_response = '' # Last continuation response + self._idle_responses = [] # Response queue for idle iteration + self._idle_capture = False # Whether to queue responses for idle + self.is_readonly = False # READ-ONLY desired state + self.tagnum = 0 + self._tls_established = False + self._mode_ascii() + self._readbuf = [] + + # Open socket to server. + + self.open(host, port, timeout) + + try: + self._connect() + except Exception: + try: + self.shutdown() + except OSError: + pass + raise + + def _mode_ascii(self): + self.utf8_enabled = False + self._encoding = 'ascii' + self.Literal = re.compile(_Literal, re.ASCII) + self.Untagged_status = re.compile(_Untagged_status, re.ASCII) + + + def _mode_utf8(self): + self.utf8_enabled = True + self._encoding = 'utf-8' + self.Literal = re.compile(_Literal) + self.Untagged_status = re.compile(_Untagged_status) + + + def _connect(self): + # Create unique tag for this session, + # and compile tagged response matcher. + + self.tagpre = Int2AP(random.randint(4096, 65535)) + self.tagre = re.compile(br'(?P' + + self.tagpre + + br'\d+) (?P[A-Z]+) (?P.*)', re.ASCII) + + # Get server welcome message, + # request and store CAPABILITY response. + + if __debug__: + self._cmd_log_len = 10 + self._cmd_log_idx = 0 + self._cmd_log = {} # Last '_cmd_log_len' interactions + if self.debug >= 1: + self._mesg('imaplib version %s' % __version__) + self._mesg('new IMAP4 connection, tag=%s' % self.tagpre) + + self.welcome = self._get_response() + if 'PREAUTH' in self.untagged_responses: + self.state = 'AUTH' + elif 'OK' in self.untagged_responses: + self.state = 'NONAUTH' + else: + raise self.error(self.welcome) + + self._get_capabilities() + if __debug__: + if self.debug >= 3: + self._mesg('CAPABILITIES: %r' % (self.capabilities,)) + + for version in AllowedVersions: + if not version in self.capabilities: + continue + self.PROTOCOL_VERSION = version + return + + raise self.error('server not IMAP4 compliant') + + + def __getattr__(self, attr): + # Allow UPPERCASE variants of IMAP4 command methods. + if attr in Commands: + return getattr(self, attr.lower()) + raise AttributeError("Unknown IMAP4 command: '%s'" % attr) + + def __enter__(self): + return self + + def __exit__(self, *args): + if self.state == "LOGOUT": + return + + try: + self.logout() + except OSError: + pass + + + # Overridable methods + + + def _create_socket(self, timeout): + # Default value of IMAP4.host is '', but socket.getaddrinfo() + # (which is used by socket.create_connection()) expects None + # as a default value for host. + if timeout is not None and not timeout: + raise ValueError('Non-blocking socket (timeout=0) is not supported') + host = None if not self.host else self.host + sys.audit("imaplib.open", self, self.host, self.port) + address = (host, self.port) + if timeout is not None: + return socket.create_connection(address, timeout) + return socket.create_connection(address) + + def open(self, host='', port=IMAP4_PORT, timeout=None): + """Setup connection to remote server on "host:port" + (default: localhost:standard IMAP4 port). + This connection will be used by the routines: + read, readline, send, shutdown. + """ + self.host = host + self.port = port + self.sock = self._create_socket(timeout) + self._file = self.sock.makefile('rb') + + + @property + def file(self): + # The old 'file' attribute is no longer used now that we do our own + # read() and readline() buffering, with which it conflicts. + # As an undocumented interface, it should never have been accessed by + # external code, and therefore does not warrant deprecation. + # Nevertheless, we provide this property for now, to avoid suddenly + # breaking any code in the wild that might have been using it in a + # harmless way. + import warnings + warnings.warn( + 'IMAP4.file is unsupported, can cause errors, and may be removed.', + RuntimeWarning, + stacklevel=2) + return self._file + + + def read(self, size): + """Read 'size' bytes from remote.""" + # We need buffered read() to continue working after socket timeouts, + # since we use them during IDLE. Unfortunately, the standard library's + # SocketIO implementation makes this impossible, by setting a permanent + # error condition instead of letting the caller decide how to handle a + # timeout. We therefore implement our own buffered read(). + # https://github.com/python/cpython/issues/51571 + # + # Reading in chunks instead of delegating to a single + # BufferedReader.read() call also means we avoid its preallocation + # of an unreasonably large memory block if a malicious server claims + # it will send a huge literal without actually sending one. + # https://github.com/python/cpython/issues/119511 + + parts = [] + + while size > 0: + + if len(parts) < len(self._readbuf): + buf = self._readbuf[len(parts)] + else: + try: + buf = self.sock.recv(DEFAULT_BUFFER_SIZE) + except ConnectionError: + break + if not buf: + break + self._readbuf.append(buf) + + if len(buf) >= size: + parts.append(buf[:size]) + self._readbuf = [buf[size:]] + self._readbuf[len(parts):] + break + parts.append(buf) + size -= len(buf) + + return b''.join(parts) + + + def readline(self): + """Read line from remote.""" + # The comment in read() explains why we implement our own readline(). + + LF = b'\n' + parts = [] + length = 0 + + while length < _MAXLINE: + + if len(parts) < len(self._readbuf): + buf = self._readbuf[len(parts)] + else: + try: + buf = self.sock.recv(DEFAULT_BUFFER_SIZE) + except ConnectionError: + break + if not buf: + break + self._readbuf.append(buf) + + pos = buf.find(LF) + if pos != -1: + pos += 1 + parts.append(buf[:pos]) + self._readbuf = [buf[pos:]] + self._readbuf[len(parts):] + break + parts.append(buf) + length += len(buf) + + line = b''.join(parts) + if len(line) > _MAXLINE: + raise self.error("got more than %d bytes" % _MAXLINE) + return line + + + def send(self, data): + """Send data to remote.""" + sys.audit("imaplib.send", self, data) + self.sock.sendall(data) + + + def shutdown(self): + """Close I/O established in "open".""" + self._file.close() + try: + self.sock.shutdown(socket.SHUT_RDWR) + except OSError as exc: + # The server might already have closed the connection. + # On Windows, this may result in WSAEINVAL (error 10022): + # An invalid operation was attempted. + if (exc.errno != errno.ENOTCONN + and getattr(exc, 'winerror', 0) != 10022): + raise + finally: + self.sock.close() + + + def socket(self): + """Return socket instance used to connect to IMAP4 server. + + socket = .socket() + """ + return self.sock + + + + # Utility methods + + + def recent(self): + """Return most recent 'RECENT' responses if any exist, + else prompt server for an update using the 'NOOP' command. + + (typ, [data]) = .recent() + + 'data' is None if no new messages, + else list of RECENT responses, most recent last. + """ + name = 'RECENT' + typ, dat = self._untagged_response('OK', [None], name) + if dat[-1]: + return typ, dat + typ, dat = self.noop() # Prod server for response + return self._untagged_response(typ, dat, name) + + + def response(self, code): + """Return data for response 'code' if received, or None. + + Old value for response 'code' is cleared. + + (code, [data]) = .response(code) + """ + return self._untagged_response(code, [None], code.upper()) + + + + # IMAP4 commands + + + def append(self, mailbox, flags, date_time, message): + """Append message to named mailbox. + + (typ, [data]) = .append(mailbox, flags, date_time, message) + + All args except 'message' can be None. + """ + name = 'APPEND' + if not mailbox: + mailbox = 'INBOX' + if flags: + if (flags[0],flags[-1]) != ('(',')'): + flags = '(%s)' % flags + else: + flags = None + if date_time: + date_time = Time2Internaldate(date_time) + else: + date_time = None + literal = MapCRLF.sub(CRLF, message) + self.literal = literal + return self._simple_command(name, mailbox, flags, date_time) + + + def authenticate(self, mechanism, authobject): + """Authenticate command - requires response processing. + + 'mechanism' specifies which authentication mechanism is to + be used - it must appear in .capabilities in the + form AUTH=. + + 'authobject' must be a callable object: + + data = authobject(response) + + It will be called to process server continuation responses; the + response argument it is passed will be a bytes. It should return bytes + data that will be base64 encoded and sent to the server. It should + return None if the client abort response '*' should be sent instead. + """ + mech = mechanism.upper() + # XXX: shouldn't this code be removed, not commented out? + #cap = 'AUTH=%s' % mech + #if not cap in self.capabilities: # Let the server decide! + # raise self.error("Server doesn't allow %s authentication." % mech) + self.literal = _Authenticator(authobject).process + typ, dat = self._simple_command('AUTHENTICATE', mech) + if typ != 'OK': + raise self.error(dat[-1].decode('utf-8', 'replace')) + self.state = 'AUTH' + return typ, dat + + + def capability(self): + """(typ, [data]) = .capability() + Fetch capabilities list from server.""" + + name = 'CAPABILITY' + typ, dat = self._simple_command(name) + return self._untagged_response(typ, dat, name) + + + def check(self): + """Checkpoint mailbox on server. + + (typ, [data]) = .check() + """ + return self._simple_command('CHECK') + + + def close(self): + """Close currently selected mailbox. + + Deleted messages are removed from writable mailbox. + This is the recommended command before 'LOGOUT'. + + (typ, [data]) = .close() + """ + try: + typ, dat = self._simple_command('CLOSE') + finally: + self.state = 'AUTH' + return typ, dat + + + def copy(self, message_set, new_mailbox): + """Copy 'message_set' messages onto end of 'new_mailbox'. + + (typ, [data]) = .copy(message_set, new_mailbox) + """ + return self._simple_command('COPY', message_set, new_mailbox) + + + def create(self, mailbox): + """Create new mailbox. + + (typ, [data]) = .create(mailbox) + """ + return self._simple_command('CREATE', mailbox) + + + def delete(self, mailbox): + """Delete old mailbox. + + (typ, [data]) = .delete(mailbox) + """ + return self._simple_command('DELETE', mailbox) + + def deleteacl(self, mailbox, who): + """Delete the ACLs (remove any rights) set for who on mailbox. + + (typ, [data]) = .deleteacl(mailbox, who) + """ + return self._simple_command('DELETEACL', mailbox, who) + + def enable(self, capability): + """Send an RFC5161 enable string to the server. + + (typ, [data]) = .enable(capability) + """ + if 'ENABLE' not in self.capabilities: + raise IMAP4.error("Server does not support ENABLE") + typ, data = self._simple_command('ENABLE', capability) + if typ == 'OK' and 'UTF8=ACCEPT' in capability.upper(): + self._mode_utf8() + return typ, data + + def expunge(self): + """Permanently remove deleted items from selected mailbox. + + Generates 'EXPUNGE' response for each deleted message. + + (typ, [data]) = .expunge() + + 'data' is list of 'EXPUNGE'd message numbers in order received. + """ + name = 'EXPUNGE' + typ, dat = self._simple_command(name) + return self._untagged_response(typ, dat, name) + + + def fetch(self, message_set, message_parts): + """Fetch (parts of) messages. + + (typ, [data, ...]) = .fetch(message_set, message_parts) + + 'message_parts' should be a string of selected parts + enclosed in parentheses, eg: "(UID BODY[TEXT])". + + 'data' are tuples of message part envelope and data. + """ + name = 'FETCH' + typ, dat = self._simple_command(name, message_set, message_parts) + return self._untagged_response(typ, dat, name) + + + def getacl(self, mailbox): + """Get the ACLs for a mailbox. + + (typ, [data]) = .getacl(mailbox) + """ + typ, dat = self._simple_command('GETACL', mailbox) + return self._untagged_response(typ, dat, 'ACL') + + + def getannotation(self, mailbox, entry, attribute): + """(typ, [data]) = .getannotation(mailbox, entry, attribute) + Retrieve ANNOTATIONs.""" + + typ, dat = self._simple_command('GETANNOTATION', mailbox, entry, attribute) + return self._untagged_response(typ, dat, 'ANNOTATION') + + + def getquota(self, root): + """Get the quota root's resource usage and limits. + + Part of the IMAP4 QUOTA extension defined in rfc2087. + + (typ, [data]) = .getquota(root) + """ + typ, dat = self._simple_command('GETQUOTA', root) + return self._untagged_response(typ, dat, 'QUOTA') + + + def getquotaroot(self, mailbox): + """Get the list of quota roots for the named mailbox. + + (typ, [[QUOTAROOT responses...], [QUOTA responses]]) = .getquotaroot(mailbox) + """ + typ, dat = self._simple_command('GETQUOTAROOT', mailbox) + typ, quota = self._untagged_response(typ, dat, 'QUOTA') + typ, quotaroot = self._untagged_response(typ, dat, 'QUOTAROOT') + return typ, [quotaroot, quota] + + + def idle(self, duration=None): + """Return an iterable IDLE context manager producing untagged responses. + If the argument is not None, limit iteration to 'duration' seconds. + + with M.idle(duration=29 * 60) as idler: + for typ, data in idler: + print(typ, data) + + Note: 'duration' requires a socket connection (not IMAP4_stream). + """ + return Idler(self, duration) + + + def list(self, directory='""', pattern='*'): + """List mailbox names in directory matching pattern. + + (typ, [data]) = .list(directory='""', pattern='*') + + 'data' is list of LIST responses. + """ + name = 'LIST' + typ, dat = self._simple_command(name, directory, pattern) + return self._untagged_response(typ, dat, name) + + + def login(self, user, password): + """Identify client using plaintext password. + + (typ, [data]) = .login(user, password) + + NB: 'password' will be quoted. + """ + typ, dat = self._simple_command('LOGIN', user, self._quote(password)) + if typ != 'OK': + raise self.error(dat[-1]) + self.state = 'AUTH' + return typ, dat + + + def login_cram_md5(self, user, password): + """ Force use of CRAM-MD5 authentication. + + (typ, [data]) = .login_cram_md5(user, password) + """ + self.user, self.password = user, password + return self.authenticate('CRAM-MD5', self._CRAM_MD5_AUTH) + + + def _CRAM_MD5_AUTH(self, challenge): + """ Authobject to use with CRAM-MD5 authentication. """ + import hmac + + if isinstance(self.password, str): + password = self.password.encode('utf-8') + else: + password = self.password + + try: + authcode = hmac.HMAC(password, challenge, 'md5') + except ValueError: # HMAC-MD5 is not available + raise self.error("CRAM-MD5 authentication is not supported") + return f"{self.user} {authcode.hexdigest()}" + + + def logout(self): + """Shutdown connection to server. + + (typ, [data]) = .logout() + + Returns server 'BYE' response. + """ + self.state = 'LOGOUT' + typ, dat = self._simple_command('LOGOUT') + self.shutdown() + return typ, dat + + + def lsub(self, directory='""', pattern='*'): + """List 'subscribed' mailbox names in directory matching pattern. + + (typ, [data, ...]) = .lsub(directory='""', pattern='*') + + 'data' are tuples of message part envelope and data. + """ + name = 'LSUB' + typ, dat = self._simple_command(name, directory, pattern) + return self._untagged_response(typ, dat, name) + + def myrights(self, mailbox): + """Show my ACLs for a mailbox (i.e. the rights that I have on mailbox). + + (typ, [data]) = .myrights(mailbox) + """ + typ,dat = self._simple_command('MYRIGHTS', mailbox) + return self._untagged_response(typ, dat, 'MYRIGHTS') + + def namespace(self): + """ Returns IMAP namespaces ala rfc2342 + + (typ, [data, ...]) = .namespace() + """ + name = 'NAMESPACE' + typ, dat = self._simple_command(name) + return self._untagged_response(typ, dat, name) + + + def noop(self): + """Send NOOP command. + + (typ, [data]) = .noop() + """ + if __debug__: + if self.debug >= 3: + self._dump_ur(self.untagged_responses) + return self._simple_command('NOOP') + + + def partial(self, message_num, message_part, start, length): + """Fetch truncated part of a message. + + (typ, [data, ...]) = .partial(message_num, message_part, start, length) + + 'data' is tuple of message part envelope and data. + """ + name = 'PARTIAL' + typ, dat = self._simple_command(name, message_num, message_part, start, length) + return self._untagged_response(typ, dat, 'FETCH') + + + def proxyauth(self, user): + """Assume authentication as "user". + + Allows an authorised administrator to proxy into any user's + mailbox. + + (typ, [data]) = .proxyauth(user) + """ + + name = 'PROXYAUTH' + return self._simple_command('PROXYAUTH', user) + + + def rename(self, oldmailbox, newmailbox): + """Rename old mailbox name to new. + + (typ, [data]) = .rename(oldmailbox, newmailbox) + """ + return self._simple_command('RENAME', oldmailbox, newmailbox) + + + def search(self, charset, *criteria): + """Search mailbox for matching messages. + + (typ, [data]) = .search(charset, criterion, ...) + + 'data' is space separated list of matching message numbers. + If UTF8 is enabled, charset MUST be None. + """ + name = 'SEARCH' + if charset: + if self.utf8_enabled: + raise IMAP4.error("Non-None charset not valid in UTF8 mode") + typ, dat = self._simple_command(name, 'CHARSET', charset, *criteria) + else: + typ, dat = self._simple_command(name, *criteria) + return self._untagged_response(typ, dat, name) + + + def select(self, mailbox='INBOX', readonly=False): + """Select a mailbox. + + Flush all untagged responses. + + (typ, [data]) = .select(mailbox='INBOX', readonly=False) + + 'data' is count of messages in mailbox ('EXISTS' response). + + Mandated responses are ('FLAGS', 'EXISTS', 'RECENT', 'UIDVALIDITY'), so + other responses should be obtained via .response('FLAGS') etc. + """ + self.untagged_responses = {} # Flush old responses. + self.is_readonly = readonly + if readonly: + name = 'EXAMINE' + else: + name = 'SELECT' + typ, dat = self._simple_command(name, mailbox) + if typ != 'OK': + self.state = 'AUTH' # Might have been 'SELECTED' + return typ, dat + self.state = 'SELECTED' + if 'READ-ONLY' in self.untagged_responses \ + and not readonly: + if __debug__: + if self.debug >= 1: + self._dump_ur(self.untagged_responses) + raise self.readonly('%s is not writable' % mailbox) + return typ, self.untagged_responses.get('EXISTS', [None]) + + + def setacl(self, mailbox, who, what): + """Set a mailbox acl. + + (typ, [data]) = .setacl(mailbox, who, what) + """ + return self._simple_command('SETACL', mailbox, who, what) + + + def setannotation(self, *args): + """(typ, [data]) = .setannotation(mailbox[, entry, attribute]+) + Set ANNOTATIONs.""" + + typ, dat = self._simple_command('SETANNOTATION', *args) + return self._untagged_response(typ, dat, 'ANNOTATION') + + + def setquota(self, root, limits): + """Set the quota root's resource limits. + + (typ, [data]) = .setquota(root, limits) + """ + typ, dat = self._simple_command('SETQUOTA', root, limits) + return self._untagged_response(typ, dat, 'QUOTA') + + + def sort(self, sort_criteria, charset, *search_criteria): + """IMAP4rev1 extension SORT command. + + (typ, [data]) = .sort(sort_criteria, charset, search_criteria, ...) + """ + name = 'SORT' + #if not name in self.capabilities: # Let the server decide! + # raise self.error('unimplemented extension command: %s' % name) + if (sort_criteria[0],sort_criteria[-1]) != ('(',')'): + sort_criteria = '(%s)' % sort_criteria + typ, dat = self._simple_command(name, sort_criteria, charset, *search_criteria) + return self._untagged_response(typ, dat, name) + + + def starttls(self, ssl_context=None): + name = 'STARTTLS' + if not HAVE_SSL: + raise self.error('SSL support missing') + if self._tls_established: + raise self.abort('TLS session already established') + if name not in self.capabilities: + raise self.abort('TLS not supported by server') + # Generate a default SSL context if none was passed. + if ssl_context is None: + ssl_context = ssl._create_stdlib_context() + typ, dat = self._simple_command(name) + if typ == 'OK': + self.sock = ssl_context.wrap_socket(self.sock, + server_hostname=self.host) + self._file = self.sock.makefile('rb') + self._tls_established = True + self._get_capabilities() + else: + raise self.error("Couldn't establish TLS session") + return self._untagged_response(typ, dat, name) + + + def status(self, mailbox, names): + """Request named status conditions for mailbox. + + (typ, [data]) = .status(mailbox, names) + """ + name = 'STATUS' + #if self.PROTOCOL_VERSION == 'IMAP4': # Let the server decide! + # raise self.error('%s unimplemented in IMAP4 (obtain IMAP4rev1 server, or re-code)' % name) + typ, dat = self._simple_command(name, mailbox, names) + return self._untagged_response(typ, dat, name) + + + def store(self, message_set, command, flags): + """Alters flag dispositions for messages in mailbox. + + (typ, [data]) = .store(message_set, command, flags) + """ + if (flags[0],flags[-1]) != ('(',')'): + flags = '(%s)' % flags # Avoid quoting the flags + typ, dat = self._simple_command('STORE', message_set, command, flags) + return self._untagged_response(typ, dat, 'FETCH') + + + def subscribe(self, mailbox): + """Subscribe to new mailbox. + + (typ, [data]) = .subscribe(mailbox) + """ + return self._simple_command('SUBSCRIBE', mailbox) + + + def thread(self, threading_algorithm, charset, *search_criteria): + """IMAPrev1 extension THREAD command. + + (type, [data]) = .thread(threading_algorithm, charset, search_criteria, ...) + """ + name = 'THREAD' + typ, dat = self._simple_command(name, threading_algorithm, charset, *search_criteria) + return self._untagged_response(typ, dat, name) + + + def uid(self, command, *args): + """Execute "command arg ..." with messages identified by UID, + rather than message number. + + (typ, [data]) = .uid(command, arg1, arg2, ...) + + Returns response appropriate to 'command'. + """ + command = command.upper() + if not command in Commands: + raise self.error("Unknown IMAP4 UID command: %s" % command) + if self.state not in Commands[command]: + raise self.error("command %s illegal in state %s, " + "only allowed in states %s" % + (command, self.state, + ', '.join(Commands[command]))) + name = 'UID' + typ, dat = self._simple_command(name, command, *args) + if command in ('SEARCH', 'SORT', 'THREAD'): + name = command + else: + name = 'FETCH' + return self._untagged_response(typ, dat, name) + + + def unsubscribe(self, mailbox): + """Unsubscribe from old mailbox. + + (typ, [data]) = .unsubscribe(mailbox) + """ + return self._simple_command('UNSUBSCRIBE', mailbox) + + + def unselect(self): + """Free server's resources associated with the selected mailbox + and returns the server to the authenticated state. + This command performs the same actions as CLOSE, except + that no messages are permanently removed from the currently + selected mailbox. + + (typ, [data]) = .unselect() + """ + try: + typ, data = self._simple_command('UNSELECT') + finally: + self.state = 'AUTH' + return typ, data + + + def xatom(self, name, *args): + """Allow simple extension commands + notified by server in CAPABILITY response. + + Assumes command is legal in current state. + + (typ, [data]) = .xatom(name, arg, ...) + + Returns response appropriate to extension command 'name'. + """ + name = name.upper() + #if not name in self.capabilities: # Let the server decide! + # raise self.error('unknown extension command: %s' % name) + if not name in Commands: + Commands[name] = (self.state,) + return self._simple_command(name, *args) + + + + # Private methods + + + def _append_untagged(self, typ, dat): + if dat is None: + dat = b'' + + # During idle, queue untagged responses for delivery via iteration + if self._idle_capture: + # Responses containing literal strings are passed to us one data + # fragment at a time, while others arrive in a single call. + if (not self._idle_responses or + isinstance(self._idle_responses[-1][1][-1], bytes)): + # We are not continuing a fragmented response; start a new one + self._idle_responses.append((typ, [dat])) + else: + # We are continuing a fragmented response; append the fragment + response = self._idle_responses[-1] + assert response[0] == typ + response[1].append(dat) + if __debug__ and self.debug >= 5: + self._mesg(f'idle: queue untagged {typ} {dat!r}') + return + + ur = self.untagged_responses + if __debug__: + if self.debug >= 5: + self._mesg('untagged_responses[%s] %s += ["%r"]' % + (typ, len(ur.get(typ,'')), dat)) + if typ in ur: + ur[typ].append(dat) + else: + ur[typ] = [dat] + + + def _check_bye(self): + bye = self.untagged_responses.get('BYE') + if bye: + raise self.abort(bye[-1].decode(self._encoding, 'replace')) + + + def _command(self, name, *args): + + if self.state not in Commands[name]: + self.literal = None + raise self.error("command %s illegal in state %s, " + "only allowed in states %s" % + (name, self.state, + ', '.join(Commands[name]))) + + for typ in ('OK', 'NO', 'BAD'): + if typ in self.untagged_responses: + del self.untagged_responses[typ] + + if 'READ-ONLY' in self.untagged_responses \ + and not self.is_readonly: + raise self.readonly('mailbox status changed to READ-ONLY') + + tag = self._new_tag() + name = bytes(name, self._encoding) + data = tag + b' ' + name + for arg in args: + if arg is None: continue + if isinstance(arg, str): + arg = bytes(arg, self._encoding) + data = data + b' ' + arg + + literal = self.literal + if literal is not None: + self.literal = None + if type(literal) is type(self._command): + literator = literal + else: + literator = None + if self.utf8_enabled: + data = data + bytes(' UTF8 (~{%s}' % len(literal), self._encoding) + literal = literal + b')' + else: + data = data + bytes(' {%s}' % len(literal), self._encoding) + + if __debug__: + if self.debug >= 4: + self._mesg('> %r' % data) + else: + self._log('> %r' % data) + + try: + self.send(data + CRLF) + except OSError as val: + raise self.abort('socket error: %s' % val) + + if literal is None: + return tag + + while 1: + # Wait for continuation response + + while self._get_response(): + if self.tagged_commands[tag]: # BAD/NO? + return tag + + # Send literal + + if literator: + literal = literator(self.continuation_response) + + if __debug__: + if self.debug >= 4: + self._mesg('write literal size %s' % len(literal)) + + try: + self.send(literal) + self.send(CRLF) + except OSError as val: + raise self.abort('socket error: %s' % val) + + if not literator: + break + + return tag + + + def _command_complete(self, name, tag): + logout = (name == 'LOGOUT') + # BYE is expected after LOGOUT + if not logout: + self._check_bye() + try: + typ, data = self._get_tagged_response(tag, expect_bye=logout) + except self.abort as val: + raise self.abort('command: %s => %s' % (name, val)) + except self.error as val: + raise self.error('command: %s => %s' % (name, val)) + if not logout: + self._check_bye() + if typ == 'BAD': + raise self.error('%s command error: %s %s' % (name, typ, data)) + return typ, data + + + def _get_capabilities(self): + typ, dat = self.capability() + if dat == [None]: + raise self.error('no CAPABILITY response from server') + dat = str(dat[-1], self._encoding) + dat = dat.upper() + self.capabilities = tuple(dat.split()) + + + def _get_response(self, start_timeout=False): + + # Read response and store. + # + # Returns None for continuation responses, + # otherwise first response line received. + # + # If start_timeout is given, temporarily uses it as a socket + # timeout while waiting for the start of a response, raising + # _responsetimeout if one doesn't arrive. (Used by Idler.) + + if start_timeout is not False and self.sock: + assert start_timeout is None or start_timeout > 0 + saved_timeout = self.sock.gettimeout() + self.sock.settimeout(start_timeout) + try: + resp = self._get_line() + except TimeoutError as err: + raise self._responsetimeout from err + finally: + self.sock.settimeout(saved_timeout) + else: + resp = self._get_line() + + # Command completion response? + + if self._match(self.tagre, resp): + tag = self.mo.group('tag') + if not tag in self.tagged_commands: + raise self.abort('unexpected tagged response: %r' % resp) + + typ = self.mo.group('type') + typ = str(typ, self._encoding) + dat = self.mo.group('data') + self.tagged_commands[tag] = (typ, [dat]) + else: + dat2 = None + + # '*' (untagged) responses? + + if not self._match(Untagged_response, resp): + if self._match(self.Untagged_status, resp): + dat2 = self.mo.group('data2') + + if self.mo is None: + # Only other possibility is '+' (continuation) response... + + if self._match(Continuation, resp): + self.continuation_response = self.mo.group('data') + return None # NB: indicates continuation + + raise self.abort("unexpected response: %r" % resp) + + typ = self.mo.group('type') + typ = str(typ, self._encoding) + dat = self.mo.group('data') + if dat is None: dat = b'' # Null untagged response + if dat2: dat = dat + b' ' + dat2 + + # Is there a literal to come? + + while self._match(self.Literal, dat): + + # Read literal direct from connection. + + size = int(self.mo.group('size')) + if __debug__: + if self.debug >= 4: + self._mesg('read literal size %s' % size) + data = self.read(size) + + # Store response with literal as tuple + + self._append_untagged(typ, (dat, data)) + + # Read trailer - possibly containing another literal + + dat = self._get_line() + + self._append_untagged(typ, dat) + + # Bracketed response information? + + if typ in ('OK', 'NO', 'BAD') and self._match(Response_code, dat): + typ = self.mo.group('type') + typ = str(typ, self._encoding) + self._append_untagged(typ, self.mo.group('data')) + + if __debug__: + if self.debug >= 1 and typ in ('NO', 'BAD', 'BYE'): + self._mesg('%s response: %r' % (typ, dat)) + + return resp + + + def _get_tagged_response(self, tag, expect_bye=False): + + while 1: + result = self.tagged_commands[tag] + if result is not None: + del self.tagged_commands[tag] + return result + + if expect_bye: + typ = 'BYE' + bye = self.untagged_responses.pop(typ, None) + if bye is not None: + # Server replies to the "LOGOUT" command with "BYE" + return (typ, bye) + + # If we've seen a BYE at this point, the socket will be + # closed, so report the BYE now. + self._check_bye() + + # Some have reported "unexpected response" exceptions. + # Note that ignoring them here causes loops. + # Instead, send me details of the unexpected response and + # I'll update the code in '_get_response()'. + + try: + self._get_response() + except self.abort as val: + if __debug__: + if self.debug >= 1: + self.print_log() + raise + + + def _get_line(self): + + line = self.readline() + if not line: + raise self.abort('socket error: EOF') + + # Protocol mandates all lines terminated by CRLF + if not line.endswith(b'\r\n'): + raise self.abort('socket error: unterminated line: %r' % line) + + line = line[:-2] + if __debug__: + if self.debug >= 4: + self._mesg('< %r' % line) + else: + self._log('< %r' % line) + return line + + + def _match(self, cre, s): + + # Run compiled regular expression match method on 's'. + # Save result, return success. + + self.mo = cre.match(s) + if __debug__: + if self.mo is not None and self.debug >= 5: + self._mesg("\tmatched %r => %r" % (cre.pattern, self.mo.groups())) + return self.mo is not None + + + def _new_tag(self): + + tag = self.tagpre + bytes(str(self.tagnum), self._encoding) + self.tagnum = self.tagnum + 1 + self.tagged_commands[tag] = None + return tag + + + def _quote(self, arg): + + arg = arg.replace('\\', '\\\\') + arg = arg.replace('"', '\\"') + + return '"' + arg + '"' + + + def _simple_command(self, name, *args): + + return self._command_complete(name, self._command(name, *args)) + + + def _untagged_response(self, typ, dat, name): + if typ == 'NO': + return typ, dat + if not name in self.untagged_responses: + return typ, [None] + data = self.untagged_responses.pop(name) + if __debug__: + if self.debug >= 5: + self._mesg('untagged_responses[%s] => %s' % (name, data)) + return typ, data + + + if __debug__: + + def _mesg(self, s, secs=None): + if secs is None: + secs = time.time() + tm = time.strftime('%M:%S', time.localtime(secs)) + sys.stderr.write(' %s.%02d %s\n' % (tm, (secs*100)%100, s)) + sys.stderr.flush() + + def _dump_ur(self, untagged_resp_dict): + if not untagged_resp_dict: + return + items = (f'{key}: {value!r}' + for key, value in untagged_resp_dict.items()) + self._mesg('untagged responses dump:' + '\n\t\t'.join(items)) + + def _log(self, line): + # Keep log of last '_cmd_log_len' interactions for debugging. + self._cmd_log[self._cmd_log_idx] = (line, time.time()) + self._cmd_log_idx += 1 + if self._cmd_log_idx >= self._cmd_log_len: + self._cmd_log_idx = 0 + + def print_log(self): + self._mesg('last %d IMAP4 interactions:' % len(self._cmd_log)) + i, n = self._cmd_log_idx, self._cmd_log_len + while n: + try: + self._mesg(*self._cmd_log[i]) + except: + pass + i += 1 + if i >= self._cmd_log_len: + i = 0 + n -= 1 + + +class Idler: + """Iterable IDLE context manager: start IDLE & produce untagged responses. + + An object of this type is returned by the IMAP4.idle() method. + + Note: The name and structure of this class are subject to change. + """ + + def __init__(self, imap, duration=None): + if 'IDLE' not in imap.capabilities: + raise imap.error("Server does not support IMAP4 IDLE") + if duration is not None and not imap.sock: + # IMAP4_stream pipes don't support timeouts + raise imap.error('duration requires a socket connection') + self._duration = duration + self._deadline = None + self._imap = imap + self._tag = None + self._saved_state = None + + def __enter__(self): + imap = self._imap + assert not imap._idle_responses + assert not imap._idle_capture + + if __debug__ and imap.debug >= 4: + imap._mesg(f'idle start duration={self._duration}') + + # Start capturing untagged responses before sending IDLE, + # so we can deliver via iteration any that arrive while + # the IDLE command continuation request is still pending. + imap._idle_capture = True + + try: + self._tag = imap._command('IDLE') + # As with any command, the server is allowed to send us unrelated, + # untagged responses before acting on IDLE. These lines will be + # returned by _get_response(). When the server is ready, it will + # send an IDLE continuation request, indicated by _get_response() + # returning None. We therefore process responses in a loop until + # this occurs. + while resp := imap._get_response(): + if imap.tagged_commands[self._tag]: + typ, data = imap.tagged_commands.pop(self._tag) + if typ == 'NO': + raise imap.error(f'idle denied: {data}') + raise imap.abort(f'unexpected status response: {resp}') + + if __debug__ and imap.debug >= 4: + prompt = imap.continuation_response + imap._mesg(f'idle continuation prompt: {prompt}') + except BaseException: + imap._idle_capture = False + raise + + if self._duration is not None: + self._deadline = time.monotonic() + self._duration + + self._saved_state = imap.state + imap.state = 'IDLING' + + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + imap = self._imap + + if __debug__ and imap.debug >= 4: + imap._mesg('idle done') + imap.state = self._saved_state + + # Stop intercepting untagged responses before sending DONE, + # since we can no longer deliver them via iteration. + imap._idle_capture = False + + # If we captured untagged responses while the IDLE command + # continuation request was still pending, but the user did not + # iterate over them before exiting IDLE, we must put them + # someplace where the user can retrieve them. The only + # sensible place for this is the untagged_responses dict, + # despite its unfortunate inability to preserve the relative + # order of different response types. + if leftovers := len(imap._idle_responses): + if __debug__ and imap.debug >= 4: + imap._mesg(f'idle quit with {leftovers} leftover responses') + while imap._idle_responses: + typ, data = imap._idle_responses.pop(0) + # Append one fragment at a time, just as _get_response() does + for datum in data: + imap._append_untagged(typ, datum) + + try: + imap.send(b'DONE' + CRLF) + status, [msg] = imap._command_complete('IDLE', self._tag) + if __debug__ and imap.debug >= 4: + imap._mesg(f'idle status: {status} {msg!r}') + except OSError: + if not exc_type: + raise + + return False # Do not suppress context body exceptions + + def __iter__(self): + return self + + def _pop(self, timeout, default=('', None)): + # Get the next response, or a default value on timeout. + # The timeout arg can be an int or float, or None for no timeout. + # Timeouts require a socket connection (not IMAP4_stream). + # This method ignores self._duration. + + # Historical Note: + # The timeout was originally implemented using select() after + # checking for the presence of already-buffered data. + # That allowed timeouts on pipe connetions like IMAP4_stream. + # However, it seemed possible that SSL data arriving without any + # IMAP data afterward could cause select() to indicate available + # application data when there was none, leading to a read() call + # that would block with no timeout. It was unclear under what + # conditions this would happen in practice. Our implementation was + # changed to use socket timeouts instead of select(), just to be + # safe. + + imap = self._imap + if imap.state != 'IDLING': + raise imap.error('_pop() only works during IDLE') + + if imap._idle_responses: + # Response is ready to return to the user + resp = imap._idle_responses.pop(0) + if __debug__ and imap.debug >= 4: + imap._mesg(f'idle _pop({timeout}) de-queued {resp[0]}') + return resp + + if __debug__ and imap.debug >= 4: + imap._mesg(f'idle _pop({timeout}) reading') + + if timeout is not None: + if timeout <= 0: + return default + timeout = float(timeout) # Required by socket.settimeout() + + try: + imap._get_response(timeout) # Reads line, calls _append_untagged() + except IMAP4._responsetimeout: + if __debug__ and imap.debug >= 4: + imap._mesg(f'idle _pop({timeout}) done') + return default + + resp = imap._idle_responses.pop(0) + + if __debug__ and imap.debug >= 4: + imap._mesg(f'idle _pop({timeout}) read {resp[0]}') + return resp + + def __next__(self): + imap = self._imap + + if self._duration is None: + timeout = None + else: + timeout = self._deadline - time.monotonic() + typ, data = self._pop(timeout) + + if not typ: + if __debug__ and imap.debug >= 4: + imap._mesg('idle iterator exhausted') + raise StopIteration + + return typ, data + + def burst(self, interval=0.1): + """Yield a burst of responses no more than 'interval' seconds apart. + + with M.idle() as idler: + # get a response and any others following by < 0.1 seconds + batch = list(idler.burst()) + print(f'processing {len(batch)} responses...') + print(batch) + + Note: This generator requires a socket connection (not IMAP4_stream). + """ + if not self._imap.sock: + raise self._imap.error('burst() requires a socket connection') + + try: + yield next(self) + except StopIteration: + return + + while response := self._pop(interval, None): + yield response + + +if HAVE_SSL: + + class IMAP4_SSL(IMAP4): + + """IMAP4 client class over SSL connection + + Instantiate with: IMAP4_SSL([host[, port[, ssl_context[, timeout=None]]]]) + + host - host's name (default: localhost); + port - port number (default: standard IMAP4 SSL port); + ssl_context - a SSLContext object that contains your certificate chain + and private key (default: None) + timeout - socket timeout (default: None) If timeout is not given or is None, + the global default socket timeout is used + + for more documentation see the docstring of the parent class IMAP4. + """ + + + def __init__(self, host='', port=IMAP4_SSL_PORT, + *, ssl_context=None, timeout=None): + if ssl_context is None: + ssl_context = ssl._create_stdlib_context() + self.ssl_context = ssl_context + IMAP4.__init__(self, host, port, timeout) + + def _create_socket(self, timeout): + sock = IMAP4._create_socket(self, timeout) + return self.ssl_context.wrap_socket(sock, + server_hostname=self.host) + + def open(self, host='', port=IMAP4_SSL_PORT, timeout=None): + """Setup connection to remote server on "host:port". + (default: localhost:standard IMAP4 SSL port). + This connection will be used by the routines: + read, readline, send, shutdown. + """ + IMAP4.open(self, host, port, timeout) + + __all__.append("IMAP4_SSL") + + +class IMAP4_stream(IMAP4): + + """IMAP4 client class over a stream + + Instantiate with: IMAP4_stream(command) + + "command" - a string that can be passed to subprocess.Popen() + + for more documentation see the docstring of the parent class IMAP4. + """ + + + def __init__(self, command): + self.command = command + IMAP4.__init__(self) + + + def open(self, host=None, port=None, timeout=None): + """Setup a stream connection. + This connection will be used by the routines: + read, readline, send, shutdown. + """ + self.host = None # For compatibility with parent class + self.port = None + self.sock = None + self._file = None + self.process = subprocess.Popen(self.command, + bufsize=DEFAULT_BUFFER_SIZE, + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + shell=True, close_fds=True) + self.writefile = self.process.stdin + self.readfile = self.process.stdout + + def read(self, size): + """Read 'size' bytes from remote.""" + return self.readfile.read(size) + + + def readline(self): + """Read line from remote.""" + return self.readfile.readline() + + + def send(self, data): + """Send data to remote.""" + self.writefile.write(data) + self.writefile.flush() + + + def shutdown(self): + """Close I/O established in "open".""" + self.readfile.close() + self.writefile.close() + self.process.wait() + + + +class _Authenticator: + + """Private class to provide en/decoding + for base64-based authentication conversation. + """ + + def __init__(self, mechinst): + self.mech = mechinst # Callable object to provide/process data + + def process(self, data): + ret = self.mech(self.decode(data)) + if ret is None: + return b'*' # Abort conversation + return self.encode(ret) + + def encode(self, inp): + # + # Invoke binascii.b2a_base64 iteratively with + # short even length buffers, strip the trailing + # line feed from the result and append. "Even" + # means a number that factors to both 6 and 8, + # so when it gets to the end of the 8-bit input + # there's no partial 6-bit output. + # + oup = b'' + if isinstance(inp, str): + inp = inp.encode('utf-8') + while inp: + if len(inp) > 48: + t = inp[:48] + inp = inp[48:] + else: + t = inp + inp = b'' + e = binascii.b2a_base64(t) + if e: + oup = oup + e[:-1] + return oup + + def decode(self, inp): + if not inp: + return b'' + return binascii.a2b_base64(inp) + +Months = ' Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec'.split(' ') +Mon2num = {s.encode():n+1 for n, s in enumerate(Months[1:])} + +def Internaldate2tuple(resp): + """Parse an IMAP4 INTERNALDATE string. + + Return corresponding local time. The return value is a + time.struct_time tuple or None if the string has wrong format. + """ + + mo = InternalDate.match(resp) + if not mo: + return None + + mon = Mon2num[mo.group('mon')] + zonen = mo.group('zonen') + + day = int(mo.group('day')) + year = int(mo.group('year')) + hour = int(mo.group('hour')) + min = int(mo.group('min')) + sec = int(mo.group('sec')) + zoneh = int(mo.group('zoneh')) + zonem = int(mo.group('zonem')) + + # INTERNALDATE timezone must be subtracted to get UT + + zone = (zoneh*60 + zonem)*60 + if zonen == b'-': + zone = -zone + + tt = (year, mon, day, hour, min, sec, -1, -1, -1) + utc = calendar.timegm(tt) - zone + + return time.localtime(utc) + + + +def Int2AP(num): + + """Convert integer to A-P string representation.""" + + val = b''; AP = b'ABCDEFGHIJKLMNOP' + num = int(abs(num)) + while num: + num, mod = divmod(num, 16) + val = AP[mod:mod+1] + val + return val + + + +def ParseFlags(resp): + + """Convert IMAP4 flags response to python tuple.""" + + mo = Flags.match(resp) + if not mo: + return () + + return tuple(mo.group('flags').split()) + + +def Time2Internaldate(date_time): + + """Convert date_time to IMAP4 INTERNALDATE representation. + + Return string in form: '"DD-Mmm-YYYY HH:MM:SS +HHMM"'. The + date_time argument can be a number (int or float) representing + seconds since epoch (as returned by time.time()), a 9-tuple + representing local time, an instance of time.struct_time (as + returned by time.localtime()), an aware datetime instance or a + double-quoted string. In the last case, it is assumed to already + be in the correct format. + """ + if isinstance(date_time, (int, float)): + dt = datetime.fromtimestamp(date_time, + timezone.utc).astimezone() + elif isinstance(date_time, tuple): + try: + gmtoff = date_time.tm_gmtoff + except AttributeError: + if time.daylight: + dst = date_time[8] + if dst == -1: + dst = time.localtime(time.mktime(date_time))[8] + gmtoff = -(time.timezone, time.altzone)[dst] + else: + gmtoff = -time.timezone + delta = timedelta(seconds=gmtoff) + dt = datetime(*date_time[:6], tzinfo=timezone(delta)) + elif isinstance(date_time, datetime): + if date_time.tzinfo is None: + raise ValueError("date_time must be aware") + dt = date_time + elif isinstance(date_time, str) and (date_time[0],date_time[-1]) == ('"','"'): + return date_time # Assume in correct format + else: + raise ValueError("date_time not of a known type") + fmt = '"%d-{}-%Y %H:%M:%S %z"'.format(Months[dt.month]) + return dt.strftime(fmt) + + + +if __name__ == '__main__': + + # To test: invoke either as 'python imaplib.py [IMAP4_server_hostname]' + # or 'python imaplib.py -s "rsh IMAP4_server_hostname exec /etc/rimapd"' + # to test the IMAP4_stream class + + import getopt, getpass + + try: + optlist, args = getopt.getopt(sys.argv[1:], 'd:s:') + except getopt.error as val: + optlist, args = (), () + + stream_command = None + for opt,val in optlist: + if opt == '-d': + Debug = int(val) + elif opt == '-s': + stream_command = val + if not args: args = (stream_command,) + + if not args: args = ('',) + + host = args[0] + + USER = getpass.getuser() + PASSWD = getpass.getpass("IMAP password for %s on %s: " % (USER, host or "localhost")) + + test_mesg = 'From: %(user)s@localhost%(lf)sSubject: IMAP4 test%(lf)s%(lf)sdata...%(lf)s' % {'user':USER, 'lf':'\n'} + test_seq1 = ( + ('login', (USER, PASSWD)), + ('create', ('/tmp/xxx 1',)), + ('rename', ('/tmp/xxx 1', '/tmp/yyy')), + ('CREATE', ('/tmp/yyz 2',)), + ('append', ('/tmp/yyz 2', None, None, test_mesg)), + ('list', ('/tmp', 'yy*')), + ('select', ('/tmp/yyz 2',)), + ('search', (None, 'SUBJECT', 'test')), + ('fetch', ('1', '(FLAGS INTERNALDATE RFC822)')), + ('store', ('1', 'FLAGS', r'(\Deleted)')), + ('namespace', ()), + ('expunge', ()), + ('recent', ()), + ('close', ()), + ) + + test_seq2 = ( + ('select', ()), + ('response',('UIDVALIDITY',)), + ('uid', ('SEARCH', 'ALL')), + ('response', ('EXISTS',)), + ('append', (None, None, None, test_mesg)), + ('recent', ()), + ('logout', ()), + ) + + def run(cmd, args): + M._mesg('%s %s' % (cmd, args)) + typ, dat = getattr(M, cmd)(*args) + M._mesg('%s => %s %s' % (cmd, typ, dat)) + if typ == 'NO': raise dat[0] + return dat + + try: + if stream_command: + M = IMAP4_stream(stream_command) + else: + M = IMAP4(host) + if M.state == 'AUTH': + test_seq1 = test_seq1[1:] # Login not needed + M._mesg('PROTOCOL_VERSION = %s' % M.PROTOCOL_VERSION) + M._mesg('CAPABILITIES = %r' % (M.capabilities,)) + + for cmd,args in test_seq1: + run(cmd, args) + + for ml in run('list', ('/tmp/', 'yy%')): + mo = re.match(r'.*"([^"]+)"$', ml) + if mo: path = mo.group(1) + else: path = ml.split()[-1] + run('delete', (path,)) + + for cmd,args in test_seq2: + dat = run(cmd, args) + + if (cmd,args) != ('uid', ('SEARCH', 'ALL')): + continue + + uid = dat[-1].split() + if not uid: continue + run('uid', ('FETCH', '%s' % uid[-1], + '(FLAGS INTERNALDATE RFC822.SIZE RFC822.HEADER RFC822.TEXT)')) + + print('\nAll tests OK.') + + except: + print('\nTests failed.') + + if not Debug: + print(''' +If you would like to see debugging output, +try: %s -d5 +''' % sys.argv[0]) + + raise diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index 68d993cacae..499da1e04ef 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -382,6 +382,9 @@ def release(self): self.waiters.pop() self.wakeup.release() + def locked(self): + return bool(self.count) + def __repr__(self): return f'_ModuleLock({self.name!r}) at {id(self)}' @@ -490,8 +493,7 @@ def _call_with_frames_removed(f, *args, **kwds): def _verbose_message(message, *args, verbosity=1): """Print the message to stderr if -v/PYTHONVERBOSE is turned on.""" - # XXX RUSTPYTHON: hasattr check because we might be bootstrapping and we wouldn't have stderr yet - if sys.flags.verbose >= verbosity and hasattr(sys, "stderr"): + if sys.flags.verbose >= verbosity: if not message.startswith(('#', 'import ')): message = '# ' + message print(message.format(*args), file=sys.stderr) @@ -1242,10 +1244,12 @@ def _find_spec(name, path, target=None): """Find a module's spec.""" meta_path = sys.meta_path if meta_path is None: - # PyImport_Cleanup() is running or has been called. raise ImportError("sys.meta_path is None, Python is likely " "shutting down") + # gh-130094: Copy sys.meta_path so that we have a consistent view of the + # list while iterating over it. + meta_path = list(meta_path) if not meta_path: _warnings.warn('sys.meta_path is empty', ImportWarning) @@ -1300,7 +1304,6 @@ def _sanity_check(name, package, level): _ERR_MSG_PREFIX = 'No module named ' -_ERR_MSG = _ERR_MSG_PREFIX + '{!r}' def _find_and_load_unlocked(name, import_): path = None @@ -1310,8 +1313,9 @@ def _find_and_load_unlocked(name, import_): if parent not in sys.modules: _call_with_frames_removed(import_, parent) # Crazy side-effects! - if name in sys.modules: - return sys.modules[name] + module = sys.modules.get(name) + if module is not None: + return module parent_module = sys.modules[parent] try: path = parent_module.__path__ @@ -1319,6 +1323,12 @@ def _find_and_load_unlocked(name, import_): msg = f'{_ERR_MSG_PREFIX}{name!r}; {parent!r} is not a package' raise ModuleNotFoundError(msg, name=name) from None parent_spec = parent_module.__spec__ + if getattr(parent_spec, '_initializing', False): + _call_with_frames_removed(import_, parent) + # Crazy side-effects (again)! + module = sys.modules.get(name) + if module is not None: + return module child = name.rpartition('.')[2] spec = _find_spec(name, path) if spec is None: diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 89ce8c09c94..95ce14b2c39 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -221,277 +221,7 @@ def _write_atomic(path, data, mode=0o666): _code_type = type(_write_atomic.__code__) - -# Finder/loader utility code ############################################### - -# Magic word to reject .pyc files generated by other Python versions. -# It should change for each incompatible change to the bytecode. -# -# The value of CR and LF is incorporated so if you ever read or write -# a .pyc file in text mode the magic number will be wrong; also, the -# Apple MPW compiler swaps their values, botching string constants. -# -# There were a variety of old schemes for setting the magic number. -# The current working scheme is to increment the previous value by -# 10. -# -# Starting with the adoption of PEP 3147 in Python 3.2, every bump in magic -# number also includes a new "magic tag", i.e. a human readable string used -# to represent the magic number in __pycache__ directories. When you change -# the magic number, you must also set a new unique magic tag. Generally this -# can be named after the Python major version of the magic number bump, but -# it can really be anything, as long as it's different than anything else -# that's come before. The tags are included in the following table, starting -# with Python 3.2a0. -# -# Known values: -# Python 1.5: 20121 -# Python 1.5.1: 20121 -# Python 1.5.2: 20121 -# Python 1.6: 50428 -# Python 2.0: 50823 -# Python 2.0.1: 50823 -# Python 2.1: 60202 -# Python 2.1.1: 60202 -# Python 2.1.2: 60202 -# Python 2.2: 60717 -# Python 2.3a0: 62011 -# Python 2.3a0: 62021 -# Python 2.3a0: 62011 (!) -# Python 2.4a0: 62041 -# Python 2.4a3: 62051 -# Python 2.4b1: 62061 -# Python 2.5a0: 62071 -# Python 2.5a0: 62081 (ast-branch) -# Python 2.5a0: 62091 (with) -# Python 2.5a0: 62092 (changed WITH_CLEANUP opcode) -# Python 2.5b3: 62101 (fix wrong code: for x, in ...) -# Python 2.5b3: 62111 (fix wrong code: x += yield) -# Python 2.5c1: 62121 (fix wrong lnotab with for loops and -# storing constants that should have been removed) -# Python 2.5c2: 62131 (fix wrong code: for x, in ... in listcomp/genexp) -# Python 2.6a0: 62151 (peephole optimizations and STORE_MAP opcode) -# Python 2.6a1: 62161 (WITH_CLEANUP optimization) -# Python 2.7a0: 62171 (optimize list comprehensions/change LIST_APPEND) -# Python 2.7a0: 62181 (optimize conditional branches: -# introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE) -# Python 2.7a0 62191 (introduce SETUP_WITH) -# Python 2.7a0 62201 (introduce BUILD_SET) -# Python 2.7a0 62211 (introduce MAP_ADD and SET_ADD) -# Python 3000: 3000 -# 3010 (removed UNARY_CONVERT) -# 3020 (added BUILD_SET) -# 3030 (added keyword-only parameters) -# 3040 (added signature annotations) -# 3050 (print becomes a function) -# 3060 (PEP 3115 metaclass syntax) -# 3061 (string literals become unicode) -# 3071 (PEP 3109 raise changes) -# 3081 (PEP 3137 make __file__ and __name__ unicode) -# 3091 (kill str8 interning) -# 3101 (merge from 2.6a0, see 62151) -# 3103 (__file__ points to source file) -# Python 3.0a4: 3111 (WITH_CLEANUP optimization). -# Python 3.0b1: 3131 (lexical exception stacking, including POP_EXCEPT - #3021) -# Python 3.1a1: 3141 (optimize list, set and dict comprehensions: -# change LIST_APPEND and SET_ADD, add MAP_ADD #2183) -# Python 3.1a1: 3151 (optimize conditional branches: -# introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE - #4715) -# Python 3.2a1: 3160 (add SETUP_WITH #6101) -# tag: cpython-32 -# Python 3.2a2: 3170 (add DUP_TOP_TWO, remove DUP_TOPX and ROT_FOUR #9225) -# tag: cpython-32 -# Python 3.2a3 3180 (add DELETE_DEREF #4617) -# Python 3.3a1 3190 (__class__ super closure changed) -# Python 3.3a1 3200 (PEP 3155 __qualname__ added #13448) -# Python 3.3a1 3210 (added size modulo 2**32 to the pyc header #13645) -# Python 3.3a2 3220 (changed PEP 380 implementation #14230) -# Python 3.3a4 3230 (revert changes to implicit __class__ closure #14857) -# Python 3.4a1 3250 (evaluate positional default arguments before -# keyword-only defaults #16967) -# Python 3.4a1 3260 (add LOAD_CLASSDEREF; allow locals of class to override -# free vars #17853) -# Python 3.4a1 3270 (various tweaks to the __class__ closure #12370) -# Python 3.4a1 3280 (remove implicit class argument) -# Python 3.4a4 3290 (changes to __qualname__ computation #19301) -# Python 3.4a4 3300 (more changes to __qualname__ computation #19301) -# Python 3.4rc2 3310 (alter __qualname__ computation #20625) -# Python 3.5a1 3320 (PEP 465: Matrix multiplication operator #21176) -# Python 3.5b1 3330 (PEP 448: Additional Unpacking Generalizations #2292) -# Python 3.5b2 3340 (fix dictionary display evaluation order #11205) -# Python 3.5b3 3350 (add GET_YIELD_FROM_ITER opcode #24400) -# Python 3.5.2 3351 (fix BUILD_MAP_UNPACK_WITH_CALL opcode #27286) -# Python 3.6a0 3360 (add FORMAT_VALUE opcode #25483) -# Python 3.6a1 3361 (lineno delta of code.co_lnotab becomes signed #26107) -# Python 3.6a2 3370 (16 bit wordcode #26647) -# Python 3.6a2 3371 (add BUILD_CONST_KEY_MAP opcode #27140) -# Python 3.6a2 3372 (MAKE_FUNCTION simplification, remove MAKE_CLOSURE -# #27095) -# Python 3.6b1 3373 (add BUILD_STRING opcode #27078) -# Python 3.6b1 3375 (add SETUP_ANNOTATIONS and STORE_ANNOTATION opcodes -# #27985) -# Python 3.6b1 3376 (simplify CALL_FUNCTIONs & BUILD_MAP_UNPACK_WITH_CALL - #27213) -# Python 3.6b1 3377 (set __class__ cell from type.__new__ #23722) -# Python 3.6b2 3378 (add BUILD_TUPLE_UNPACK_WITH_CALL #28257) -# Python 3.6rc1 3379 (more thorough __class__ validation #23722) -# Python 3.7a1 3390 (add LOAD_METHOD and CALL_METHOD opcodes #26110) -# Python 3.7a2 3391 (update GET_AITER #31709) -# Python 3.7a4 3392 (PEP 552: Deterministic pycs #31650) -# Python 3.7b1 3393 (remove STORE_ANNOTATION opcode #32550) -# Python 3.7b5 3394 (restored docstring as the first stmt in the body; -# this might affected the first line number #32911) -# Python 3.8a1 3400 (move frame block handling to compiler #17611) -# Python 3.8a1 3401 (add END_ASYNC_FOR #33041) -# Python 3.8a1 3410 (PEP570 Python Positional-Only Parameters #36540) -# Python 3.8b2 3411 (Reverse evaluation order of key: value in dict -# comprehensions #35224) -# Python 3.8b2 3412 (Swap the position of positional args and positional -# only args in ast.arguments #37593) -# Python 3.8b4 3413 (Fix "break" and "continue" in "finally" #37830) -# Python 3.9a0 3420 (add LOAD_ASSERTION_ERROR #34880) -# Python 3.9a0 3421 (simplified bytecode for with blocks #32949) -# Python 3.9a0 3422 (remove BEGIN_FINALLY, END_FINALLY, CALL_FINALLY, POP_FINALLY bytecodes #33387) -# Python 3.9a2 3423 (add IS_OP, CONTAINS_OP and JUMP_IF_NOT_EXC_MATCH bytecodes #39156) -# Python 3.9a2 3424 (simplify bytecodes for *value unpacking) -# Python 3.9a2 3425 (simplify bytecodes for **value unpacking) -# Python 3.10a1 3430 (Make 'annotations' future by default) -# Python 3.10a1 3431 (New line number table format -- PEP 626) -# Python 3.10a2 3432 (Function annotation for MAKE_FUNCTION is changed from dict to tuple bpo-42202) -# Python 3.10a2 3433 (RERAISE restores f_lasti if oparg != 0) -# Python 3.10a6 3434 (PEP 634: Structural Pattern Matching) -# Python 3.10a7 3435 Use instruction offsets (as opposed to byte offsets). -# Python 3.10b1 3436 (Add GEN_START bytecode #43683) -# Python 3.10b1 3437 (Undo making 'annotations' future by default - We like to dance among core devs!) -# Python 3.10b1 3438 Safer line number table handling. -# Python 3.10b1 3439 (Add ROT_N) -# Python 3.11a1 3450 Use exception table for unwinding ("zero cost" exception handling) -# Python 3.11a1 3451 (Add CALL_METHOD_KW) -# Python 3.11a1 3452 (drop nlocals from marshaled code objects) -# Python 3.11a1 3453 (add co_fastlocalnames and co_fastlocalkinds) -# Python 3.11a1 3454 (compute cell offsets relative to locals bpo-43693) -# Python 3.11a1 3455 (add MAKE_CELL bpo-43693) -# Python 3.11a1 3456 (interleave cell args bpo-43693) -# Python 3.11a1 3457 (Change localsplus to a bytes object bpo-43693) -# Python 3.11a1 3458 (imported objects now don't use LOAD_METHOD/CALL_METHOD) -# Python 3.11a1 3459 (PEP 657: add end line numbers and column offsets for instructions) -# Python 3.11a1 3460 (Add co_qualname field to PyCodeObject bpo-44530) -# Python 3.11a1 3461 (JUMP_ABSOLUTE must jump backwards) -# Python 3.11a2 3462 (bpo-44511: remove COPY_DICT_WITHOUT_KEYS, change -# MATCH_CLASS and MATCH_KEYS, and add COPY) -# Python 3.11a3 3463 (bpo-45711: JUMP_IF_NOT_EXC_MATCH no longer pops the -# active exception) -# Python 3.11a3 3464 (bpo-45636: Merge numeric BINARY_*/INPLACE_* into -# BINARY_OP) -# Python 3.11a3 3465 (Add COPY_FREE_VARS opcode) -# Python 3.11a4 3466 (bpo-45292: PEP-654 except*) -# Python 3.11a4 3467 (Change CALL_xxx opcodes) -# Python 3.11a4 3468 (Add SEND opcode) -# Python 3.11a4 3469 (bpo-45711: remove type, traceback from exc_info) -# Python 3.11a4 3470 (bpo-46221: PREP_RERAISE_STAR no longer pushes lasti) -# Python 3.11a4 3471 (bpo-46202: remove pop POP_EXCEPT_AND_RERAISE) -# Python 3.11a4 3472 (bpo-46009: replace GEN_START with POP_TOP) -# Python 3.11a4 3473 (Add POP_JUMP_IF_NOT_NONE/POP_JUMP_IF_NONE opcodes) -# Python 3.11a4 3474 (Add RESUME opcode) -# Python 3.11a5 3475 (Add RETURN_GENERATOR opcode) -# Python 3.11a5 3476 (Add ASYNC_GEN_WRAP opcode) -# Python 3.11a5 3477 (Replace DUP_TOP/DUP_TOP_TWO with COPY and -# ROT_TWO/ROT_THREE/ROT_FOUR/ROT_N with SWAP) -# Python 3.11a5 3478 (New CALL opcodes) -# Python 3.11a5 3479 (Add PUSH_NULL opcode) -# Python 3.11a5 3480 (New CALL opcodes, second iteration) -# Python 3.11a5 3481 (Use inline cache for BINARY_OP) -# Python 3.11a5 3482 (Use inline caching for UNPACK_SEQUENCE and LOAD_GLOBAL) -# Python 3.11a5 3483 (Use inline caching for COMPARE_OP and BINARY_SUBSCR) -# Python 3.11a5 3484 (Use inline caching for LOAD_ATTR, LOAD_METHOD, and -# STORE_ATTR) -# Python 3.11a5 3485 (Add an oparg to GET_AWAITABLE) -# Python 3.11a6 3486 (Use inline caching for PRECALL and CALL) -# Python 3.11a6 3487 (Remove the adaptive "oparg counter" mechanism) -# Python 3.11a6 3488 (LOAD_GLOBAL can push additional NULL) -# Python 3.11a6 3489 (Add JUMP_BACKWARD, remove JUMP_ABSOLUTE) -# Python 3.11a6 3490 (remove JUMP_IF_NOT_EXC_MATCH, add CHECK_EXC_MATCH) -# Python 3.11a6 3491 (remove JUMP_IF_NOT_EG_MATCH, add CHECK_EG_MATCH, -# add JUMP_BACKWARD_NO_INTERRUPT, make JUMP_NO_INTERRUPT virtual) -# Python 3.11a7 3492 (make POP_JUMP_IF_NONE/NOT_NONE/TRUE/FALSE relative) -# Python 3.11a7 3493 (Make JUMP_IF_TRUE_OR_POP/JUMP_IF_FALSE_OR_POP relative) -# Python 3.11a7 3494 (New location info table) -# Python 3.11b4 3495 (Set line number of module's RESUME instr to 0 per PEP 626) -# Python 3.12a1 3500 (Remove PRECALL opcode) -# Python 3.12a1 3501 (YIELD_VALUE oparg == stack_depth) -# Python 3.12a1 3502 (LOAD_FAST_CHECK, no NULL-check in LOAD_FAST) -# Python 3.12a1 3503 (Shrink LOAD_METHOD cache) -# Python 3.12a1 3504 (Merge LOAD_METHOD back into LOAD_ATTR) -# Python 3.12a1 3505 (Specialization/Cache for FOR_ITER) -# Python 3.12a1 3506 (Add BINARY_SLICE and STORE_SLICE instructions) -# Python 3.12a1 3507 (Set lineno of module's RESUME to 0) -# Python 3.12a1 3508 (Add CLEANUP_THROW) -# Python 3.12a1 3509 (Conditional jumps only jump forward) -# Python 3.12a2 3510 (FOR_ITER leaves iterator on the stack) -# Python 3.12a2 3511 (Add STOPITERATION_ERROR instruction) -# Python 3.12a2 3512 (Remove all unused consts from code objects) -# Python 3.12a4 3513 (Add CALL_INTRINSIC_1 instruction, removed STOPITERATION_ERROR, PRINT_EXPR, IMPORT_STAR) -# Python 3.12a4 3514 (Remove ASYNC_GEN_WRAP, LIST_TO_TUPLE, and UNARY_POSITIVE) -# Python 3.12a5 3515 (Embed jump mask in COMPARE_OP oparg) -# Python 3.12a5 3516 (Add COMPARE_AND_BRANCH instruction) -# Python 3.12a5 3517 (Change YIELD_VALUE oparg to exception block depth) -# Python 3.12a6 3518 (Add RETURN_CONST instruction) -# Python 3.12a6 3519 (Modify SEND instruction) -# Python 3.12a6 3520 (Remove PREP_RERAISE_STAR, add CALL_INTRINSIC_2) -# Python 3.12a7 3521 (Shrink the LOAD_GLOBAL caches) -# Python 3.12a7 3522 (Removed JUMP_IF_FALSE_OR_POP/JUMP_IF_TRUE_OR_POP) -# Python 3.12a7 3523 (Convert COMPARE_AND_BRANCH back to COMPARE_OP) -# Python 3.12a7 3524 (Shrink the BINARY_SUBSCR caches) -# Python 3.12b1 3525 (Shrink the CALL caches) -# Python 3.12b1 3526 (Add instrumentation support) -# Python 3.12b1 3527 (Add LOAD_SUPER_ATTR) -# Python 3.12b1 3528 (Add LOAD_SUPER_ATTR_METHOD specialization) -# Python 3.12b1 3529 (Inline list/dict/set comprehensions) -# Python 3.12b1 3530 (Shrink the LOAD_SUPER_ATTR caches) -# Python 3.12b1 3531 (Add PEP 695 changes) -# Python 3.13a1 3550 (Plugin optimizer support) -# Python 3.13a1 3551 (Compact superinstructions) -# Python 3.13a1 3552 (Remove LOAD_FAST__LOAD_CONST and LOAD_CONST__LOAD_FAST) -# Python 3.13a1 3553 (Add SET_FUNCTION_ATTRIBUTE) -# Python 3.13a1 3554 (more efficient bytecodes for f-strings) -# Python 3.13a1 3555 (generate specialized opcodes metadata from bytecodes.c) -# Python 3.13a1 3556 (Convert LOAD_CLOSURE to a pseudo-op) -# Python 3.13a1 3557 (Make the conversion to boolean in jumps explicit) -# Python 3.13a1 3558 (Reorder the stack items for CALL) -# Python 3.13a1 3559 (Generate opcode IDs from bytecodes.c) -# Python 3.13a1 3560 (Add RESUME_CHECK instruction) -# Python 3.13a1 3561 (Add cache entry to branch instructions) -# Python 3.13a1 3562 (Assign opcode IDs for internal ops in separate range) -# Python 3.13a1 3563 (Add CALL_KW and remove KW_NAMES) -# Python 3.13a1 3564 (Removed oparg from YIELD_VALUE, changed oparg values of RESUME) -# Python 3.13a1 3565 (Oparg of YIELD_VALUE indicates whether it is in a yield-from) -# Python 3.13a1 3566 (Emit JUMP_NO_INTERRUPT instead of JUMP for non-loop no-lineno cases) -# Python 3.13a1 3567 (Reimplement line number propagation by the compiler) -# Python 3.13a1 3568 (Change semantics of END_FOR) -# Python 3.13a5 3569 (Specialize CONTAINS_OP) -# Python 3.13a6 3570 (Add __firstlineno__ class attribute) -# Python 3.13b1 3571 (Fix miscompilation of private names in generic classes) - -# Python 3.14 will start with 3600 - -# Please don't copy-paste the same pre-release tag for new entries above!!! -# You should always use the *upcoming* tag. For example, if 3.12a6 came out -# a week ago, I should put "Python 3.12a7" next to my new magic number. - -# MAGIC must change whenever the bytecode emitted by the compiler may no -# longer be understood by older implementations of the eval loop (usually -# due to the addition of new opcodes). -# -# Starting with Python 3.11, Python 3.n starts with magic number 2900+50n. -# -# Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array -# in PC/launcher.c must also be updated. - -MAGIC_NUMBER = (2996).to_bytes(2, 'little') + b'\r\n' - -_RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c +MAGIC_NUMBER = _imp.pyc_magic_number_token.to_bytes(4, 'little') _PYCACHE = '__pycache__' _OPT = 'opt-' @@ -983,6 +713,12 @@ def _search_registry(cls, fullname): @classmethod def find_spec(cls, fullname, path=None, target=None): + _warnings.warn('importlib.machinery.WindowsRegistryFinder is ' + 'deprecated; use site configuration instead. ' + 'Future versions of Python may not enable this ' + 'finder by default.', + DeprecationWarning, stacklevel=2) + filepath = cls._search_registry(fullname) if filepath is None: return None @@ -1131,7 +867,7 @@ def get_code(self, fullname): _imp.check_hash_based_pycs == 'always')): source_bytes = self.get_data(source_path) source_hash = _imp.source_hash( - _RAW_MAGIC_NUMBER, + _imp.pyc_magic_number_token, source_bytes, ) _validate_hash_pyc(data, source_hash, fullname, @@ -1160,7 +896,7 @@ def get_code(self, fullname): source_mtime is not None): if hash_based: if source_hash is None: - source_hash = _imp.source_hash(_RAW_MAGIC_NUMBER, + source_hash = _imp.source_hash(_imp.pyc_magic_number_token, source_bytes) data = _code_to_hash_pyc(code_object, source_hash, check_source) else: @@ -1505,7 +1241,7 @@ def _path_importer_cache(cls, path): if path == '': try: path = _os.getcwd() - except FileNotFoundError: + except (FileNotFoundError, PermissionError): # Don't cache the failure as the cwd can easily change to # a valid directory later on. return None diff --git a/Lib/importlib/abc.py b/Lib/importlib/abc.py index 37fef357fe2..1e47495f65f 100644 --- a/Lib/importlib/abc.py +++ b/Lib/importlib/abc.py @@ -13,9 +13,6 @@ _frozen_importlib_external = _bootstrap_external from ._abc import Loader import abc -import warnings - -from .resources import abc as _resources_abc __all__ = [ @@ -25,19 +22,6 @@ ] -def __getattr__(name): - """ - For backwards compatibility, continue to make names - from _resources_abc available through this module. #93963 - """ - if name in _resources_abc.__all__: - obj = getattr(_resources_abc, name) - warnings._deprecated(f"{__name__}.{name}", remove=(3, 14)) - globals()[name] = obj - return obj - raise AttributeError(f'module {__name__!r} has no attribute {name!r}') - - def _register(abstract_cls, *classes): for cls in classes: abstract_cls.register(cls) @@ -80,10 +64,13 @@ def invalidate_caches(self): class ResourceLoader(Loader): """Abstract base class for loaders which can return data from their - back-end storage. + back-end storage to facilitate reading data to perform an import. This ABC represents one of the optional protocols specified by PEP 302. + For directly loading resources, use TraversableResources instead. This class + primarily exists for backwards compatibility with other ABCs in this module. + """ @abc.abstractmethod @@ -215,6 +202,10 @@ class SourceLoader(_bootstrap_external.SourceLoader, ResourceLoader, ExecutionLo def path_mtime(self, path): """Return the (int) modification time for the path (str).""" + import warnings + warnings.warn('SourceLoader.path_mtime is deprecated in favour of ' + 'SourceLoader.path_stats().', + DeprecationWarning, stacklevel=2) if self.path_stats.__func__ is SourceLoader.path_stats: raise OSError return int(self.path_stats(path)['mtime']) diff --git a/Lib/importlib/machinery.py b/Lib/importlib/machinery.py index fbd30b159fb..63d726445c3 100644 --- a/Lib/importlib/machinery.py +++ b/Lib/importlib/machinery.py @@ -3,9 +3,11 @@ from ._bootstrap import ModuleSpec from ._bootstrap import BuiltinImporter from ._bootstrap import FrozenImporter -from ._bootstrap_external import (SOURCE_SUFFIXES, DEBUG_BYTECODE_SUFFIXES, - OPTIMIZED_BYTECODE_SUFFIXES, BYTECODE_SUFFIXES, - EXTENSION_SUFFIXES) +from ._bootstrap_external import ( + SOURCE_SUFFIXES, BYTECODE_SUFFIXES, EXTENSION_SUFFIXES, + DEBUG_BYTECODE_SUFFIXES as _DEBUG_BYTECODE_SUFFIXES, + OPTIMIZED_BYTECODE_SUFFIXES as _OPTIMIZED_BYTECODE_SUFFIXES +) from ._bootstrap_external import WindowsRegistryFinder from ._bootstrap_external import PathFinder from ._bootstrap_external import FileFinder @@ -19,3 +21,30 @@ def all_suffixes(): """Returns a list of all recognized module suffixes for this process""" return SOURCE_SUFFIXES + BYTECODE_SUFFIXES + EXTENSION_SUFFIXES + + +__all__ = ['AppleFrameworkLoader', 'BYTECODE_SUFFIXES', 'BuiltinImporter', + 'DEBUG_BYTECODE_SUFFIXES', 'EXTENSION_SUFFIXES', + 'ExtensionFileLoader', 'FileFinder', 'FrozenImporter', 'ModuleSpec', + 'NamespaceLoader', 'OPTIMIZED_BYTECODE_SUFFIXES', 'PathFinder', + 'SOURCE_SUFFIXES', 'SourceFileLoader', 'SourcelessFileLoader', + 'WindowsRegistryFinder', 'all_suffixes'] + + +def __getattr__(name): + import warnings + + if name == 'DEBUG_BYTECODE_SUFFIXES': + warnings.warn('importlib.machinery.DEBUG_BYTECODE_SUFFIXES is ' + 'deprecated; use importlib.machinery.BYTECODE_SUFFIXES ' + 'instead.', + DeprecationWarning, stacklevel=2) + return _DEBUG_BYTECODE_SUFFIXES + elif name == 'OPTIMIZED_BYTECODE_SUFFIXES': + warnings.warn('importlib.machinery.OPTIMIZED_BYTECODE_SUFFIXES is ' + 'deprecated; use importlib.machinery.BYTECODE_SUFFIXES ' + 'instead.', + DeprecationWarning, stacklevel=2) + return _OPTIMIZED_BYTECODE_SUFFIXES + + raise AttributeError(f'module {__name__!r} has no attribute {name!r}') diff --git a/Lib/importlib/resources/_common.py b/Lib/importlib/resources/_common.py index d9128266a2e..4e9014c45a0 100644 --- a/Lib/importlib/resources/_common.py +++ b/Lib/importlib/resources/_common.py @@ -77,12 +77,12 @@ def resolve(cand: Optional[Anchor]) -> types.ModuleType: return cast(types.ModuleType, cand) -@resolve.register(str) # TODO: RUSTPYTHON; manual type annotation +@resolve.register def _(cand: str) -> types.ModuleType: return importlib.import_module(cand) -@resolve.register(type(None)) # TODO: RUSTPYTHON; manual type annotation +@resolve.register def _(cand: None) -> types.ModuleType: return resolve(_infer_caller().f_globals['__name__']) @@ -183,7 +183,7 @@ def _(path): @contextlib.contextmanager def _temp_path(dir: tempfile.TemporaryDirectory): """ - Wrap tempfile.TemporyDirectory to return a pathlib object. + Wrap tempfile.TemporaryDirectory to return a pathlib object. """ with dir as result: yield pathlib.Path(result) diff --git a/Lib/importlib/resources/_legacy.py b/Lib/importlib/resources/_legacy.py deleted file mode 100644 index b1ea8105dad..00000000000 --- a/Lib/importlib/resources/_legacy.py +++ /dev/null @@ -1,120 +0,0 @@ -import functools -import os -import pathlib -import types -import warnings - -from typing import Union, Iterable, ContextManager, BinaryIO, TextIO, Any - -from . import _common - -Package = Union[types.ModuleType, str] -Resource = str - - -def deprecated(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - warnings.warn( - f"{func.__name__} is deprecated. Use files() instead. " - "Refer to https://importlib-resources.readthedocs.io" - "/en/latest/using.html#migrating-from-legacy for migration advice.", - DeprecationWarning, - stacklevel=2, - ) - return func(*args, **kwargs) - - return wrapper - - -def normalize_path(path: Any) -> str: - """Normalize a path by ensuring it is a string. - - If the resulting string contains path separators, an exception is raised. - """ - str_path = str(path) - parent, file_name = os.path.split(str_path) - if parent: - raise ValueError(f'{path!r} must be only a file name') - return file_name - - -@deprecated -def open_binary(package: Package, resource: Resource) -> BinaryIO: - """Return a file-like object opened for binary reading of the resource.""" - return (_common.files(package) / normalize_path(resource)).open('rb') - - -@deprecated -def read_binary(package: Package, resource: Resource) -> bytes: - """Return the binary contents of the resource.""" - return (_common.files(package) / normalize_path(resource)).read_bytes() - - -@deprecated -def open_text( - package: Package, - resource: Resource, - encoding: str = 'utf-8', - errors: str = 'strict', -) -> TextIO: - """Return a file-like object opened for text reading of the resource.""" - return (_common.files(package) / normalize_path(resource)).open( - 'r', encoding=encoding, errors=errors - ) - - -@deprecated -def read_text( - package: Package, - resource: Resource, - encoding: str = 'utf-8', - errors: str = 'strict', -) -> str: - """Return the decoded string of the resource. - - The decoding-related arguments have the same semantics as those of - bytes.decode(). - """ - with open_text(package, resource, encoding, errors) as fp: - return fp.read() - - -@deprecated -def contents(package: Package) -> Iterable[str]: - """Return an iterable of entries in `package`. - - Note that not all entries are resources. Specifically, directories are - not considered resources. Use `is_resource()` on each entry returned here - to check if it is a resource or not. - """ - return [path.name for path in _common.files(package).iterdir()] - - -@deprecated -def is_resource(package: Package, name: str) -> bool: - """True if `name` is a resource inside `package`. - - Directories are *not* resources. - """ - resource = normalize_path(name) - return any( - traversable.name == resource and traversable.is_file() - for traversable in _common.files(package).iterdir() - ) - - -@deprecated -def path( - package: Package, - resource: Resource, -) -> ContextManager[pathlib.Path]: - """A context manager providing a file path object to the resource. - - If the resource does not already exist on its own on the file system, - a temporary file will be created. If the file was created, the file - will be deleted upon exiting the context manager (no exception is - raised if the file was deleted prior to the context manager - exiting). - """ - return _common.as_file(_common.files(package) / normalize_path(resource)) diff --git a/Lib/importlib/util.py b/Lib/importlib/util.py index 284206b62f9..2b564e9b52e 100644 --- a/Lib/importlib/util.py +++ b/Lib/importlib/util.py @@ -5,7 +5,6 @@ from ._bootstrap import spec_from_loader from ._bootstrap import _find_spec from ._bootstrap_external import MAGIC_NUMBER -from ._bootstrap_external import _RAW_MAGIC_NUMBER from ._bootstrap_external import cache_from_source from ._bootstrap_external import decode_source from ._bootstrap_external import source_from_cache @@ -18,7 +17,7 @@ def source_hash(source_bytes): "Return the hash of *source_bytes* as used in hash-based pyc files." - return _imp.source_hash(_RAW_MAGIC_NUMBER, source_bytes) + return _imp.source_hash(_imp.pyc_magic_number_token, source_bytes) def resolve_name(name, package): @@ -272,3 +271,9 @@ def exec_module(self, module): loader_state['is_loading'] = False module.__spec__.loader_state = loader_state module.__class__ = _LazyModule + + +__all__ = ['LazyLoader', 'Loader', 'MAGIC_NUMBER', + 'cache_from_source', 'decode_source', 'find_spec', + 'module_from_spec', 'resolve_name', 'source_from_cache', + 'source_hash', 'spec_from_file_location', 'spec_from_loader'] diff --git a/Lib/inspect.py b/Lib/inspect.py index 385fbc686b6..3cee85f39a6 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -348,6 +348,7 @@ def isgenerator(object): gi_frame frame object or possibly None once the generator has been exhausted gi_running set to 1 when generator is executing, 0 otherwise + gi_suspended set to 1 when the generator is suspended at a yield point, 0 otherwise gi_yieldfrom object being iterated by yield from or None __iter__() defined to support iteration over container diff --git a/Lib/io.py b/Lib/io.py index f0e2fa15d5a..63ffadb1d38 100644 --- a/Lib/io.py +++ b/Lib/io.py @@ -46,21 +46,20 @@ "BufferedReader", "BufferedWriter", "BufferedRWPair", "BufferedRandom", "TextIOBase", "TextIOWrapper", "UnsupportedOperation", "SEEK_SET", "SEEK_CUR", "SEEK_END", - "DEFAULT_BUFFER_SIZE", "text_encoding", "IncrementalNewlineDecoder"] + "DEFAULT_BUFFER_SIZE", "text_encoding", "IncrementalNewlineDecoder", + "Reader", "Writer"] import _io import abc +from _collections_abc import _check_methods from _io import (DEFAULT_BUFFER_SIZE, BlockingIOError, UnsupportedOperation, open, open_code, FileIO, BytesIO, StringIO, BufferedReader, BufferedWriter, BufferedRWPair, BufferedRandom, IncrementalNewlineDecoder, text_encoding, TextIOWrapper) -# Pretend this exception was created here. -UnsupportedOperation.__module__ = "io" - # for seek() SEEK_SET = 0 SEEK_CUR = 1 @@ -97,3 +96,55 @@ class TextIOBase(_io._TextIOBase, IOBase): pass else: RawIOBase.register(_WindowsConsoleIO) + +# +# Static Typing Support +# + +GenericAlias = type(list[int]) + + +class Reader(metaclass=abc.ABCMeta): + """Protocol for simple I/O reader instances. + + This protocol only supports blocking I/O. + """ + + __slots__ = () + + @abc.abstractmethod + def read(self, size=..., /): + """Read data from the input stream and return it. + + If *size* is specified, at most *size* items (bytes/characters) will be + read. + """ + + @classmethod + def __subclasshook__(cls, C): + if cls is Reader: + return _check_methods(C, "read") + return NotImplemented + + __class_getitem__ = classmethod(GenericAlias) + + +class Writer(metaclass=abc.ABCMeta): + """Protocol for simple I/O writer instances. + + This protocol only supports blocking I/O. + """ + + __slots__ = () + + @abc.abstractmethod + def write(self, data, /): + """Write *data* to the output stream and return the number of items written.""" + + @classmethod + def __subclasshook__(cls, C): + if cls is Writer: + return _check_methods(C, "write") + return NotImplemented + + __class_getitem__ = classmethod(GenericAlias) diff --git a/Lib/linecache.py b/Lib/linecache.py index 2b5a31b3e75..ef3b2d9136b 100644 --- a/Lib/linecache.py +++ b/Lib/linecache.py @@ -33,10 +33,9 @@ def getlines(filename, module_globals=None): """Get the lines for a Python source file from the cache. Update the cache if it doesn't contain an entry for this file already.""" - if filename in cache: - entry = cache[filename] - if len(entry) != 1: - return cache[filename][2] + entry = cache.get(filename, None) + if entry is not None and len(entry) != 1: + return entry[2] try: return updatecache(filename, module_globals) @@ -56,10 +55,9 @@ def _make_key(code): def _getlines_from_code(code): code_id = _make_key(code) - if code_id in _interactive_cache: - entry = _interactive_cache[code_id] - if len(entry) != 1: - return _interactive_cache[code_id][2] + entry = _interactive_cache.get(code_id, None) + if entry is not None and len(entry) != 1: + return entry[2] return [] @@ -84,12 +82,8 @@ def checkcache(filename=None): filenames = [filename] for filename in filenames: - try: - entry = cache[filename] - except KeyError: - continue - - if len(entry) == 1: + entry = cache.get(filename, None) + if entry is None or len(entry) == 1: # lazy cache entry, leave it lazy. continue size, mtime, lines, fullname = entry @@ -125,9 +119,7 @@ def updatecache(filename, module_globals=None): # These import can fail if the interpreter is shutting down return [] - if filename in cache: - if len(cache[filename]) != 1: - cache.pop(filename, None) + entry = cache.pop(filename, None) if _source_unavailable(filename): return [] @@ -149,9 +141,12 @@ def updatecache(filename, module_globals=None): # Realise a lazy loader based lookup if there is one # otherwise try to lookup right now. - if lazycache(filename, module_globals): + lazy_entry = entry if entry is not None and len(entry) == 1 else None + if lazy_entry is None: + lazy_entry = _make_lazycache_entry(filename, module_globals) + if lazy_entry is not None: try: - data = cache[filename][0]() + data = lazy_entry[0]() except (ImportError, OSError): pass else: @@ -159,13 +154,14 @@ def updatecache(filename, module_globals=None): # No luck, the PEP302 loader cannot find the source # for this module. return [] - cache[filename] = ( + entry = ( len(data), None, [line + '\n' for line in data.splitlines()], fullname ) - return cache[filename][2] + cache[filename] = entry + return entry[2] # Try looking through the module search path, which is only useful # when handling a relative filename. @@ -214,13 +210,20 @@ def lazycache(filename, module_globals): get_source method must be found, the filename must be a cacheable filename, and the filename must not be already cached. """ - if filename in cache: - if len(cache[filename]) == 1: - return True - else: - return False + entry = cache.get(filename, None) + if entry is not None: + return len(entry) == 1 + + lazy_entry = _make_lazycache_entry(filename, module_globals) + if lazy_entry is not None: + cache[filename] = lazy_entry + return True + return False + + +def _make_lazycache_entry(filename, module_globals): if not filename or (filename.startswith('<') and filename.endswith('>')): - return False + return None # Try for a __loader__, if available if module_globals and '__name__' in module_globals: spec = module_globals.get('__spec__') @@ -233,9 +236,10 @@ def lazycache(filename, module_globals): if name and get_source: def get_lines(name=name, *args, **kwargs): return get_source(name, *args, **kwargs) - cache[filename] = (get_lines,) - return True - return False + return (get_lines,) + return None + + def _register_code(code, string, name): entry = (len(string), @@ -248,4 +252,5 @@ def _register_code(code, string, name): for const in code.co_consts: if isinstance(const, type(code)): stack.append(const) - _interactive_cache[_make_key(code)] = entry + key = _make_key(code) + _interactive_cache[key] = entry diff --git a/Lib/locale.py b/Lib/locale.py index db6d0abb26b..dfedc6386cb 100644 --- a/Lib/locale.py +++ b/Lib/locale.py @@ -13,7 +13,6 @@ import sys import encodings import encodings.aliases -import re import _collections_abc from builtins import str as _builtin_str import functools @@ -177,8 +176,7 @@ def _strip_padding(s, amount): amount -= 1 return s[lpos:rpos+1] -_percent_re = re.compile(r'%(?:\((?P.*?)\))?' - r'(?P[-#0-9 +*.hlL]*?)[eEfFgGdiouxXcrs%]') +_percent_re = None def _format(percent, value, grouping=False, monetary=False, *additional): if additional: @@ -217,6 +215,13 @@ def format_string(f, val, grouping=False, monetary=False): Grouping is applied if the third parameter is true. Conversion uses monetary thousands separator and grouping strings if forth parameter monetary is true.""" + global _percent_re + if _percent_re is None: + import re + + _percent_re = re.compile(r'%(?:\((?P.*?)\))?(?P[-#0-9 +*.hlL]*?)[eEfFgGdiouxXcrs%]') + percents = list(_percent_re.finditer(f)) new_f = _percent_re.sub('%s', f) diff --git a/Lib/mailbox.py b/Lib/mailbox.py index b00d9e8634c..364af6bb010 100644 --- a/Lib/mailbox.py +++ b/Lib/mailbox.py @@ -2183,11 +2183,7 @@ def _unlock_file(f): def _create_carefully(path): """Create a file if it doesn't exist and open for reading and writing.""" - fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0o666) - try: - return open(path, 'rb+') - finally: - os.close(fd) + return open(path, 'xb+') def _create_temporary(path): """Create a temp file based on path and open for reading and writing.""" diff --git a/Lib/modulefinder.py b/Lib/modulefinder.py new file mode 100644 index 00000000000..ac478ee7f51 --- /dev/null +++ b/Lib/modulefinder.py @@ -0,0 +1,671 @@ +"""Find modules used by a script, using introspection.""" + +import dis +import importlib._bootstrap_external +import importlib.machinery +import marshal +import os +import io +import sys + +# Old imp constants: + +_SEARCH_ERROR = 0 +_PY_SOURCE = 1 +_PY_COMPILED = 2 +_C_EXTENSION = 3 +_PKG_DIRECTORY = 5 +_C_BUILTIN = 6 +_PY_FROZEN = 7 + +# Modulefinder does a good job at simulating Python's, but it can not +# handle __path__ modifications packages make at runtime. Therefore there +# is a mechanism whereby you can register extra paths in this map for a +# package, and it will be honored. + +# Note this is a mapping is lists of paths. +packagePathMap = {} + +# A Public interface +def AddPackagePath(packagename, path): + packagePathMap.setdefault(packagename, []).append(path) + +replacePackageMap = {} + +# This ReplacePackage mechanism allows modulefinder to work around +# situations in which a package injects itself under the name +# of another package into sys.modules at runtime by calling +# ReplacePackage("real_package_name", "faked_package_name") +# before running ModuleFinder. + +def ReplacePackage(oldname, newname): + replacePackageMap[oldname] = newname + + +def _find_module(name, path=None): + """An importlib reimplementation of imp.find_module (for our purposes).""" + + # It's necessary to clear the caches for our Finder first, in case any + # modules are being added/deleted/modified at runtime. In particular, + # test_modulefinder.py changes file tree contents in a cache-breaking way: + + importlib.machinery.PathFinder.invalidate_caches() + + spec = importlib.machinery.PathFinder.find_spec(name, path) + + if spec is None: + raise ImportError("No module named {name!r}".format(name=name), name=name) + + # Some special cases: + + if spec.loader is importlib.machinery.BuiltinImporter: + return None, None, ("", "", _C_BUILTIN) + + if spec.loader is importlib.machinery.FrozenImporter: + return None, None, ("", "", _PY_FROZEN) + + file_path = spec.origin + + if spec.loader.is_package(name): + return None, os.path.dirname(file_path), ("", "", _PKG_DIRECTORY) + + if isinstance(spec.loader, importlib.machinery.SourceFileLoader): + kind = _PY_SOURCE + + elif isinstance( + spec.loader, ( + importlib.machinery.ExtensionFileLoader, + importlib.machinery.AppleFrameworkLoader, + ) + ): + kind = _C_EXTENSION + + elif isinstance(spec.loader, importlib.machinery.SourcelessFileLoader): + kind = _PY_COMPILED + + else: # Should never happen. + return None, None, ("", "", _SEARCH_ERROR) + + file = io.open_code(file_path) + suffix = os.path.splitext(file_path)[-1] + + return file, file_path, (suffix, "rb", kind) + + +class Module: + + def __init__(self, name, file=None, path=None): + self.__name__ = name + self.__file__ = file + self.__path__ = path + self.__code__ = None + # The set of global names that are assigned to in the module. + # This includes those names imported through starimports of + # Python modules. + self.globalnames = {} + # The set of starimports this module did that could not be + # resolved, ie. a starimport from a non-Python module. + self.starimports = {} + + def __repr__(self): + s = "Module(%r" % (self.__name__,) + if self.__file__ is not None: + s = s + ", %r" % (self.__file__,) + if self.__path__ is not None: + s = s + ", %r" % (self.__path__,) + s = s + ")" + return s + +class ModuleFinder: + + def __init__(self, path=None, debug=0, excludes=None, replace_paths=None): + if path is None: + path = sys.path + self.path = path + self.modules = {} + self.badmodules = {} + self.debug = debug + self.indent = 0 + self.excludes = excludes if excludes is not None else [] + self.replace_paths = replace_paths if replace_paths is not None else [] + self.processed_paths = [] # Used in debugging only + + def msg(self, level, str, *args): + if level <= self.debug: + for i in range(self.indent): + print(" ", end=' ') + print(str, end=' ') + for arg in args: + print(repr(arg), end=' ') + print() + + def msgin(self, *args): + level = args[0] + if level <= self.debug: + self.indent = self.indent + 1 + self.msg(*args) + + def msgout(self, *args): + level = args[0] + if level <= self.debug: + self.indent = self.indent - 1 + self.msg(*args) + + def run_script(self, pathname): + self.msg(2, "run_script", pathname) + with io.open_code(pathname) as fp: + stuff = ("", "rb", _PY_SOURCE) + self.load_module('__main__', fp, pathname, stuff) + + def load_file(self, pathname): + dir, name = os.path.split(pathname) + name, ext = os.path.splitext(name) + with io.open_code(pathname) as fp: + stuff = (ext, "rb", _PY_SOURCE) + self.load_module(name, fp, pathname, stuff) + + def import_hook(self, name, caller=None, fromlist=None, level=-1): + self.msg(3, "import_hook", name, caller, fromlist, level) + parent = self.determine_parent(caller, level=level) + q, tail = self.find_head_package(parent, name) + m = self.load_tail(q, tail) + if not fromlist: + return q + if m.__path__: + self.ensure_fromlist(m, fromlist) + return None + + def determine_parent(self, caller, level=-1): + self.msgin(4, "determine_parent", caller, level) + if not caller or level == 0: + self.msgout(4, "determine_parent -> None") + return None + pname = caller.__name__ + if level >= 1: # relative import + if caller.__path__: + level -= 1 + if level == 0: + parent = self.modules[pname] + assert parent is caller + self.msgout(4, "determine_parent ->", parent) + return parent + if pname.count(".") < level: + raise ImportError("relative importpath too deep") + pname = ".".join(pname.split(".")[:-level]) + parent = self.modules[pname] + self.msgout(4, "determine_parent ->", parent) + return parent + if caller.__path__: + parent = self.modules[pname] + assert caller is parent + self.msgout(4, "determine_parent ->", parent) + return parent + if '.' in pname: + i = pname.rfind('.') + pname = pname[:i] + parent = self.modules[pname] + assert parent.__name__ == pname + self.msgout(4, "determine_parent ->", parent) + return parent + self.msgout(4, "determine_parent -> None") + return None + + def find_head_package(self, parent, name): + self.msgin(4, "find_head_package", parent, name) + if '.' in name: + i = name.find('.') + head = name[:i] + tail = name[i+1:] + else: + head = name + tail = "" + if parent: + qname = "%s.%s" % (parent.__name__, head) + else: + qname = head + q = self.import_module(head, qname, parent) + if q: + self.msgout(4, "find_head_package ->", (q, tail)) + return q, tail + if parent: + qname = head + parent = None + q = self.import_module(head, qname, parent) + if q: + self.msgout(4, "find_head_package ->", (q, tail)) + return q, tail + self.msgout(4, "raise ImportError: No module named", qname) + raise ImportError("No module named " + qname) + + def load_tail(self, q, tail): + self.msgin(4, "load_tail", q, tail) + m = q + while tail: + i = tail.find('.') + if i < 0: i = len(tail) + head, tail = tail[:i], tail[i+1:] + mname = "%s.%s" % (m.__name__, head) + m = self.import_module(head, mname, m) + if not m: + self.msgout(4, "raise ImportError: No module named", mname) + raise ImportError("No module named " + mname) + self.msgout(4, "load_tail ->", m) + return m + + def ensure_fromlist(self, m, fromlist, recursive=0): + self.msg(4, "ensure_fromlist", m, fromlist, recursive) + for sub in fromlist: + if sub == "*": + if not recursive: + all = self.find_all_submodules(m) + if all: + self.ensure_fromlist(m, all, 1) + elif not hasattr(m, sub): + subname = "%s.%s" % (m.__name__, sub) + submod = self.import_module(sub, subname, m) + if not submod: + raise ImportError("No module named " + subname) + + def find_all_submodules(self, m): + if not m.__path__: + return + modules = {} + # 'suffixes' used to be a list hardcoded to [".py", ".pyc"]. + # But we must also collect Python extension modules - although + # we cannot separate normal dlls from Python extensions. + suffixes = [] + suffixes += importlib.machinery.EXTENSION_SUFFIXES[:] + suffixes += importlib.machinery.SOURCE_SUFFIXES[:] + suffixes += importlib.machinery.BYTECODE_SUFFIXES[:] + for dir in m.__path__: + try: + names = os.listdir(dir) + except OSError: + self.msg(2, "can't list directory", dir) + continue + for name in names: + mod = None + for suff in suffixes: + n = len(suff) + if name[-n:] == suff: + mod = name[:-n] + break + if mod and mod != "__init__": + modules[mod] = mod + return modules.keys() + + def import_module(self, partname, fqname, parent): + self.msgin(3, "import_module", partname, fqname, parent) + try: + m = self.modules[fqname] + except KeyError: + pass + else: + self.msgout(3, "import_module ->", m) + return m + if fqname in self.badmodules: + self.msgout(3, "import_module -> None") + return None + if parent and parent.__path__ is None: + self.msgout(3, "import_module -> None") + return None + try: + fp, pathname, stuff = self.find_module(partname, + parent and parent.__path__, parent) + except ImportError: + self.msgout(3, "import_module ->", None) + return None + + try: + m = self.load_module(fqname, fp, pathname, stuff) + finally: + if fp: + fp.close() + if parent: + setattr(parent, partname, m) + self.msgout(3, "import_module ->", m) + return m + + def load_module(self, fqname, fp, pathname, file_info): + suffix, mode, type = file_info + self.msgin(2, "load_module", fqname, fp and "fp", pathname) + if type == _PKG_DIRECTORY: + m = self.load_package(fqname, pathname) + self.msgout(2, "load_module ->", m) + return m + if type == _PY_SOURCE: + co = compile(fp.read(), pathname, 'exec') + elif type == _PY_COMPILED: + try: + data = fp.read() + importlib._bootstrap_external._classify_pyc(data, fqname, {}) + except ImportError as exc: + self.msgout(2, "raise ImportError: " + str(exc), pathname) + raise + co = marshal.loads(memoryview(data)[16:]) + else: + co = None + m = self.add_module(fqname) + m.__file__ = pathname + if co: + if self.replace_paths: + co = self.replace_paths_in_code(co) + m.__code__ = co + self.scan_code(co, m) + self.msgout(2, "load_module ->", m) + return m + + def _add_badmodule(self, name, caller): + if name not in self.badmodules: + self.badmodules[name] = {} + if caller: + self.badmodules[name][caller.__name__] = 1 + else: + self.badmodules[name]["-"] = 1 + + def _safe_import_hook(self, name, caller, fromlist, level=-1): + # wrapper for self.import_hook() that won't raise ImportError + if name in self.badmodules: + self._add_badmodule(name, caller) + return + try: + self.import_hook(name, caller, level=level) + except ImportError as msg: + self.msg(2, "ImportError:", str(msg)) + self._add_badmodule(name, caller) + except SyntaxError as msg: + self.msg(2, "SyntaxError:", str(msg)) + self._add_badmodule(name, caller) + else: + if fromlist: + for sub in fromlist: + fullname = name + "." + sub + if fullname in self.badmodules: + self._add_badmodule(fullname, caller) + continue + try: + self.import_hook(name, caller, [sub], level=level) + except ImportError as msg: + self.msg(2, "ImportError:", str(msg)) + self._add_badmodule(fullname, caller) + + def scan_opcodes(self, co): + # Scan the code, and yield 'interesting' opcode combinations + for name in dis._find_store_names(co): + yield "store", (name,) + for name, level, fromlist in dis._find_imports(co): + if level == 0: # absolute import + yield "absolute_import", (fromlist, name) + else: # relative import + yield "relative_import", (level, fromlist, name) + + def scan_code(self, co, m): + code = co.co_code + scanner = self.scan_opcodes + for what, args in scanner(co): + if what == "store": + name, = args + m.globalnames[name] = 1 + elif what == "absolute_import": + fromlist, name = args + have_star = 0 + if fromlist is not None: + if "*" in fromlist: + have_star = 1 + fromlist = [f for f in fromlist if f != "*"] + self._safe_import_hook(name, m, fromlist, level=0) + if have_star: + # We've encountered an "import *". If it is a Python module, + # the code has already been parsed and we can suck out the + # global names. + mm = None + if m.__path__: + # At this point we don't know whether 'name' is a + # submodule of 'm' or a global module. Let's just try + # the full name first. + mm = self.modules.get(m.__name__ + "." + name) + if mm is None: + mm = self.modules.get(name) + if mm is not None: + m.globalnames.update(mm.globalnames) + m.starimports.update(mm.starimports) + if mm.__code__ is None: + m.starimports[name] = 1 + else: + m.starimports[name] = 1 + elif what == "relative_import": + level, fromlist, name = args + if name: + self._safe_import_hook(name, m, fromlist, level=level) + else: + parent = self.determine_parent(m, level=level) + self._safe_import_hook(parent.__name__, None, fromlist, level=0) + else: + # We don't expect anything else from the generator. + raise RuntimeError(what) + + for c in co.co_consts: + if isinstance(c, type(co)): + self.scan_code(c, m) + + def load_package(self, fqname, pathname): + self.msgin(2, "load_package", fqname, pathname) + newname = replacePackageMap.get(fqname) + if newname: + fqname = newname + m = self.add_module(fqname) + m.__file__ = pathname + m.__path__ = [pathname] + + # As per comment at top of file, simulate runtime __path__ additions. + m.__path__ = m.__path__ + packagePathMap.get(fqname, []) + + fp, buf, stuff = self.find_module("__init__", m.__path__) + try: + self.load_module(fqname, fp, buf, stuff) + self.msgout(2, "load_package ->", m) + return m + finally: + if fp: + fp.close() + + def add_module(self, fqname): + if fqname in self.modules: + return self.modules[fqname] + self.modules[fqname] = m = Module(fqname) + return m + + def find_module(self, name, path, parent=None): + if parent is not None: + # assert path is not None + fullname = parent.__name__+'.'+name + else: + fullname = name + if fullname in self.excludes: + self.msgout(3, "find_module -> Excluded", fullname) + raise ImportError(name) + + if path is None: + if name in sys.builtin_module_names: + return (None, None, ("", "", _C_BUILTIN)) + + path = self.path + + return _find_module(name, path) + + def report(self): + """Print a report to stdout, listing the found modules with their + paths, as well as modules that are missing, or seem to be missing. + """ + print() + print(" %-25s %s" % ("Name", "File")) + print(" %-25s %s" % ("----", "----")) + # Print modules found + keys = sorted(self.modules.keys()) + for key in keys: + m = self.modules[key] + if m.__path__: + print("P", end=' ') + else: + print("m", end=' ') + print("%-25s" % key, m.__file__ or "") + + # Print missing modules + missing, maybe = self.any_missing_maybe() + if missing: + print() + print("Missing modules:") + for name in missing: + mods = sorted(self.badmodules[name].keys()) + print("?", name, "imported from", ', '.join(mods)) + # Print modules that may be missing, but then again, maybe not... + if maybe: + print() + print("Submodules that appear to be missing, but could also be", end=' ') + print("global names in the parent package:") + for name in maybe: + mods = sorted(self.badmodules[name].keys()) + print("?", name, "imported from", ', '.join(mods)) + + def any_missing(self): + """Return a list of modules that appear to be missing. Use + any_missing_maybe() if you want to know which modules are + certain to be missing, and which *may* be missing. + """ + missing, maybe = self.any_missing_maybe() + return missing + maybe + + def any_missing_maybe(self): + """Return two lists, one with modules that are certainly missing + and one with modules that *may* be missing. The latter names could + either be submodules *or* just global names in the package. + + The reason it can't always be determined is that it's impossible to + tell which names are imported when "from module import *" is done + with an extension module, short of actually importing it. + """ + missing = [] + maybe = [] + for name in self.badmodules: + if name in self.excludes: + continue + i = name.rfind(".") + if i < 0: + missing.append(name) + continue + subname = name[i+1:] + pkgname = name[:i] + pkg = self.modules.get(pkgname) + if pkg is not None: + if pkgname in self.badmodules[name]: + # The package tried to import this module itself and + # failed. It's definitely missing. + missing.append(name) + elif subname in pkg.globalnames: + # It's a global in the package: definitely not missing. + pass + elif pkg.starimports: + # It could be missing, but the package did an "import *" + # from a non-Python module, so we simply can't be sure. + maybe.append(name) + else: + # It's not a global in the package, the package didn't + # do funny star imports, it's very likely to be missing. + # The symbol could be inserted into the package from the + # outside, but since that's not good style we simply list + # it missing. + missing.append(name) + else: + missing.append(name) + missing.sort() + maybe.sort() + return missing, maybe + + def replace_paths_in_code(self, co): + new_filename = original_filename = os.path.normpath(co.co_filename) + for f, r in self.replace_paths: + if original_filename.startswith(f): + new_filename = r + original_filename[len(f):] + break + + if self.debug and original_filename not in self.processed_paths: + if new_filename != original_filename: + self.msgout(2, "co_filename %r changed to %r" \ + % (original_filename,new_filename,)) + else: + self.msgout(2, "co_filename %r remains unchanged" \ + % (original_filename,)) + self.processed_paths.append(original_filename) + + consts = list(co.co_consts) + for i in range(len(consts)): + if isinstance(consts[i], type(co)): + consts[i] = self.replace_paths_in_code(consts[i]) + + return co.replace(co_consts=tuple(consts), co_filename=new_filename) + + +def test(): + # Parse command line + import getopt + try: + opts, args = getopt.getopt(sys.argv[1:], "dmp:qx:") + except getopt.error as msg: + print(msg) + return + + # Process options + debug = 1 + domods = 0 + addpath = [] + exclude = [] + for o, a in opts: + if o == '-d': + debug = debug + 1 + if o == '-m': + domods = 1 + if o == '-p': + addpath = addpath + a.split(os.pathsep) + if o == '-q': + debug = 0 + if o == '-x': + exclude.append(a) + + # Provide default arguments + if not args: + script = "hello.py" + else: + script = args[0] + + # Set the path based on sys.path and the script directory + path = sys.path[:] + path[0] = os.path.dirname(script) + path = addpath + path + if debug > 1: + print("path:") + for item in path: + print(" ", repr(item)) + + # Create the module finder and turn its crank + mf = ModuleFinder(path, debug, exclude) + for arg in args[1:]: + if arg == '-m': + domods = 1 + continue + if domods: + if arg[-2:] == '.*': + mf.import_hook(arg[:-2], None, ["*"]) + else: + mf.import_hook(arg) + else: + mf.load_file(arg) + mf.run_script(script) + mf.report() + return mf # for -i debugging + + +if __name__ == '__main__': + try: + mf = test() + except KeyboardInterrupt: + print("\n[interrupted]") diff --git a/Lib/multiprocessing/util.py b/Lib/multiprocessing/util.py index b8bfea045df..4c8425064fe 100644 --- a/Lib/multiprocessing/util.py +++ b/Lib/multiprocessing/util.py @@ -517,15 +517,13 @@ def _flush_std_streams(): def spawnv_passfds(path, args, passfds): import _posixsubprocess - import subprocess passfds = tuple(sorted(map(int, passfds))) errpipe_read, errpipe_write = os.pipe() try: return _posixsubprocess.fork_exec( args, [path], True, passfds, None, None, -1, -1, -1, -1, -1, -1, errpipe_read, errpipe_write, - False, False, -1, None, None, None, -1, None, - subprocess._USE_VFORK) + False, False, -1, None, None, None, -1, None) finally: os.close(errpipe_read) os.close(errpipe_write) diff --git a/Lib/optparse.py b/Lib/optparse.py index 1c450c6fcbe..38cf16d21ef 100644 --- a/Lib/optparse.py +++ b/Lib/optparse.py @@ -43,7 +43,7 @@ __copyright__ = """ Copyright (c) 2001-2006 Gregory P. Ward. All rights reserved. -Copyright (c) 2002-2006 Python Software Foundation. All rights reserved. +Copyright (c) 2002 Python Software Foundation. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -74,7 +74,8 @@ """ import sys, os -import textwrap +from gettext import gettext as _, ngettext + def _repr(self): return "<%s at 0x%x: %s>" % (self.__class__.__name__, id(self), self) @@ -86,19 +87,6 @@ def _repr(self): # Id: help.py 527 2006-07-23 15:21:30Z greg # Id: errors.py 509 2006-04-20 00:58:24Z gward -try: - from gettext import gettext, ngettext -except ImportError: - def gettext(message): - return message - - def ngettext(singular, plural, n): - if n == 1: - return singular - return plural - -_ = gettext - class OptParseError (Exception): def __init__(self, msg): @@ -263,6 +251,7 @@ def _format_text(self, text): Format a paragraph of free-form text for inclusion in the help output at the current indentation level. """ + import textwrap text_width = max(self.width - self.current_indent, 11) indent = " "*self.current_indent return textwrap.fill(text, @@ -319,6 +308,7 @@ def format_option(self, option): indent_first = 0 result.append(opts) if option.help: + import textwrap help_text = self.expand_default(option) help_lines = textwrap.wrap(help_text, self.help_width) result.append("%*s%s\n" % (indent_first, "", help_lines[0])) diff --git a/Lib/os.py b/Lib/os.py index b4c9f84c36d..ac03b416390 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -10,7 +10,7 @@ - os.extsep is the extension separator (always '.') - os.altsep is the alternate pathname separator (None or '/') - os.pathsep is the component separator used in $PATH etc - - os.linesep is the line separator in text files ('\r' or '\n' or '\r\n') + - os.linesep is the line separator in text files ('\n' or '\r\n') - os.defpath is the default search path for executables - os.devnull is the file path of the null device ('/dev/null', etc.) @@ -64,6 +64,10 @@ def _get_exports_list(module): from posix import _have_functions except ImportError: pass + try: + from posix import _create_environ + except ImportError: + pass import posix __all__.extend(_get_exports_list(posix)) @@ -88,6 +92,10 @@ def _get_exports_list(module): from nt import _have_functions except ImportError: pass + try: + from nt import _create_environ + except ImportError: + pass else: raise ImportError('no os specific module found') @@ -366,61 +374,45 @@ def walk(top, topdown=True, onerror=None, followlinks=False): # minor reason when (say) a thousand readable directories are still # left to visit. try: - scandir_it = scandir(top) + with scandir(top) as entries: + for entry in entries: + try: + if followlinks is _walk_symlinks_as_files: + is_dir = entry.is_dir(follow_symlinks=False) and not entry.is_junction() + else: + is_dir = entry.is_dir() + except OSError: + # If is_dir() raises an OSError, consider the entry not to + # be a directory, same behaviour as os.path.isdir(). + is_dir = False + + if is_dir: + dirs.append(entry.name) + else: + nondirs.append(entry.name) + + if not topdown and is_dir: + # Bottom-up: traverse into sub-directory, but exclude + # symlinks to directories if followlinks is False + if followlinks: + walk_into = True + else: + try: + is_symlink = entry.is_symlink() + except OSError: + # If is_symlink() raises an OSError, consider the + # entry not to be a symbolic link, same behaviour + # as os.path.islink(). + is_symlink = False + walk_into = not is_symlink + + if walk_into: + walk_dirs.append(entry.path) except OSError as error: if onerror is not None: onerror(error) continue - cont = False - with scandir_it: - while True: - try: - try: - entry = next(scandir_it) - except StopIteration: - break - except OSError as error: - if onerror is not None: - onerror(error) - cont = True - break - - try: - if followlinks is _walk_symlinks_as_files: - is_dir = entry.is_dir(follow_symlinks=False) and not entry.is_junction() - else: - is_dir = entry.is_dir() - except OSError: - # If is_dir() raises an OSError, consider the entry not to - # be a directory, same behaviour as os.path.isdir(). - is_dir = False - - if is_dir: - dirs.append(entry.name) - else: - nondirs.append(entry.name) - - if not topdown and is_dir: - # Bottom-up: traverse into sub-directory, but exclude - # symlinks to directories if followlinks is False - if followlinks: - walk_into = True - else: - try: - is_symlink = entry.is_symlink() - except OSError: - # If is_symlink() raises an OSError, consider the - # entry not to be a symbolic link, same behaviour - # as os.path.islink(). - is_symlink = False - walk_into = not is_symlink - - if walk_into: - walk_dirs.append(entry.path) - if cont: - continue - if topdown: # Yield before sub-directory traversal if going top down yield top, dirs, nondirs @@ -774,7 +766,7 @@ def __ror__(self, other): new.update(self) return new -def _createenviron(): +def _create_environ_mapping(): if name == 'nt': # Where Env Var Names Must Be UPPERCASE def check_str(value): @@ -804,9 +796,24 @@ def decode(value): encode, decode) # unicode environ -environ = _createenviron() -del _createenviron +environ = _create_environ_mapping() +del _create_environ_mapping + + +if _exists("_create_environ"): + def reload_environ(): + data = _create_environ() + if name == 'nt': + encodekey = environ.encodekey + data = {encodekey(key): value + for key, value in data.items()} + + # modify in-place to keep os.environb in sync + env_data = environ._data + env_data.clear() + env_data.update(data) + __all__.append("reload_environ") def getenv(key, default=None): """Get an environment variable, return None if it doesn't exist. diff --git a/Lib/pathlib/__init__.py b/Lib/pathlib/__init__.py index 4b3edf535a6..0d763d1f0dc 100644 --- a/Lib/pathlib/__init__.py +++ b/Lib/pathlib/__init__.py @@ -5,8 +5,1303 @@ operating systems. """ -from ._abc import * -from ._local import * +import io +import ntpath +import operator +import os +import posixpath +import sys +from errno import * +from glob import _StringGlobber, _no_recurse_symlinks +from itertools import chain +from stat import S_ISDIR, S_ISREG, S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO +from _collections_abc import Sequence -__all__ = (_abc.__all__ + - _local.__all__) +try: + import pwd +except ImportError: + pwd = None +try: + import grp +except ImportError: + grp = None + +from pathlib._os import ( + PathInfo, DirEntryInfo, + ensure_different_files, ensure_distinct_paths, + copyfile2, copyfileobj, magic_open, copy_info, +) + + +__all__ = [ + "UnsupportedOperation", + "PurePath", "PurePosixPath", "PureWindowsPath", + "Path", "PosixPath", "WindowsPath", + ] + + +class UnsupportedOperation(NotImplementedError): + """An exception that is raised when an unsupported operation is attempted. + """ + pass + + +class _PathParents(Sequence): + """This object provides sequence-like access to the logical ancestors + of a path. Don't try to construct it yourself.""" + __slots__ = ('_path', '_drv', '_root', '_tail') + + def __init__(self, path): + self._path = path + self._drv = path.drive + self._root = path.root + self._tail = path._tail + + def __len__(self): + return len(self._tail) + + def __getitem__(self, idx): + if isinstance(idx, slice): + return tuple(self[i] for i in range(*idx.indices(len(self)))) + + if idx >= len(self) or idx < -len(self): + raise IndexError(idx) + if idx < 0: + idx += len(self) + return self._path._from_parsed_parts(self._drv, self._root, + self._tail[:-idx - 1]) + + def __repr__(self): + return "<{}.parents>".format(type(self._path).__name__) + + +class PurePath: + """Base class for manipulating paths without I/O. + + PurePath represents a filesystem path and offers operations which + don't imply any actual filesystem I/O. Depending on your system, + instantiating a PurePath will return either a PurePosixPath or a + PureWindowsPath object. You can also instantiate either of these classes + directly, regardless of your system. + """ + + __slots__ = ( + # The `_raw_paths` slot stores unjoined string paths. This is set in + # the `__init__()` method. + '_raw_paths', + + # The `_drv`, `_root` and `_tail_cached` slots store parsed and + # normalized parts of the path. They are set when any of the `drive`, + # `root` or `_tail` properties are accessed for the first time. The + # three-part division corresponds to the result of + # `os.path.splitroot()`, except that the tail is further split on path + # separators (i.e. it is a list of strings), and that the root and + # tail are normalized. + '_drv', '_root', '_tail_cached', + + # The `_str` slot stores the string representation of the path, + # computed from the drive, root and tail when `__str__()` is called + # for the first time. It's used to implement `_str_normcase` + '_str', + + # The `_str_normcase_cached` slot stores the string path with + # normalized case. It is set when the `_str_normcase` property is + # accessed for the first time. It's used to implement `__eq__()` + # `__hash__()`, and `_parts_normcase` + '_str_normcase_cached', + + # The `_parts_normcase_cached` slot stores the case-normalized + # string path after splitting on path separators. It's set when the + # `_parts_normcase` property is accessed for the first time. It's used + # to implement comparison methods like `__lt__()`. + '_parts_normcase_cached', + + # The `_hash` slot stores the hash of the case-normalized string + # path. It's set when `__hash__()` is called for the first time. + '_hash', + ) + parser = os.path + + def __new__(cls, *args, **kwargs): + """Construct a PurePath from one or several strings and or existing + PurePath objects. The strings and path objects are combined so as + to yield a canonicalized path, which is incorporated into the + new PurePath object. + """ + if cls is PurePath: + cls = PureWindowsPath if os.name == 'nt' else PurePosixPath + return object.__new__(cls) + + def __init__(self, *args): + paths = [] + for arg in args: + if isinstance(arg, PurePath): + if arg.parser is not self.parser: + # GH-103631: Convert separators for backwards compatibility. + paths.append(arg.as_posix()) + else: + paths.extend(arg._raw_paths) + else: + try: + path = os.fspath(arg) + except TypeError: + path = arg + if not isinstance(path, str): + raise TypeError( + "argument should be a str or an os.PathLike " + "object where __fspath__ returns a str, " + f"not {type(path).__name__!r}") + paths.append(path) + self._raw_paths = paths + + def with_segments(self, *pathsegments): + """Construct a new path object from any number of path-like objects. + Subclasses may override this method to customize how new path objects + are created from methods like `iterdir()`. + """ + return type(self)(*pathsegments) + + def joinpath(self, *pathsegments): + """Combine this path with one or several arguments, and return a + new path representing either a subpath (if all arguments are relative + paths) or a totally different path (if one of the arguments is + anchored). + """ + return self.with_segments(self, *pathsegments) + + def __truediv__(self, key): + try: + return self.with_segments(self, key) + except TypeError: + return NotImplemented + + def __rtruediv__(self, key): + try: + return self.with_segments(key, self) + except TypeError: + return NotImplemented + + def __reduce__(self): + return self.__class__, tuple(self._raw_paths) + + def __repr__(self): + return "{}({!r})".format(self.__class__.__name__, self.as_posix()) + + def __fspath__(self): + return str(self) + + def __bytes__(self): + """Return the bytes representation of the path. This is only + recommended to use under Unix.""" + return os.fsencode(self) + + @property + def _str_normcase(self): + # String with normalized case, for hashing and equality checks + try: + return self._str_normcase_cached + except AttributeError: + if self.parser is posixpath: + self._str_normcase_cached = str(self) + else: + self._str_normcase_cached = str(self).lower() + return self._str_normcase_cached + + def __hash__(self): + try: + return self._hash + except AttributeError: + self._hash = hash(self._str_normcase) + return self._hash + + def __eq__(self, other): + if not isinstance(other, PurePath): + return NotImplemented + return self._str_normcase == other._str_normcase and self.parser is other.parser + + @property + def _parts_normcase(self): + # Cached parts with normalized case, for comparisons. + try: + return self._parts_normcase_cached + except AttributeError: + self._parts_normcase_cached = self._str_normcase.split(self.parser.sep) + return self._parts_normcase_cached + + def __lt__(self, other): + if not isinstance(other, PurePath) or self.parser is not other.parser: + return NotImplemented + return self._parts_normcase < other._parts_normcase + + def __le__(self, other): + if not isinstance(other, PurePath) or self.parser is not other.parser: + return NotImplemented + return self._parts_normcase <= other._parts_normcase + + def __gt__(self, other): + if not isinstance(other, PurePath) or self.parser is not other.parser: + return NotImplemented + return self._parts_normcase > other._parts_normcase + + def __ge__(self, other): + if not isinstance(other, PurePath) or self.parser is not other.parser: + return NotImplemented + return self._parts_normcase >= other._parts_normcase + + def __str__(self): + """Return the string representation of the path, suitable for + passing to system calls.""" + try: + return self._str + except AttributeError: + self._str = self._format_parsed_parts(self.drive, self.root, + self._tail) or '.' + return self._str + + @classmethod + def _format_parsed_parts(cls, drv, root, tail): + if drv or root: + return drv + root + cls.parser.sep.join(tail) + elif tail and cls.parser.splitdrive(tail[0])[0]: + tail = ['.'] + tail + return cls.parser.sep.join(tail) + + def _from_parsed_parts(self, drv, root, tail): + path = self._from_parsed_string(self._format_parsed_parts(drv, root, tail)) + path._drv = drv + path._root = root + path._tail_cached = tail + return path + + def _from_parsed_string(self, path_str): + path = self.with_segments(path_str) + path._str = path_str or '.' + return path + + @classmethod + def _parse_path(cls, path): + if not path: + return '', '', [] + sep = cls.parser.sep + altsep = cls.parser.altsep + if altsep: + path = path.replace(altsep, sep) + drv, root, rel = cls.parser.splitroot(path) + if not root and drv.startswith(sep) and not drv.endswith(sep): + drv_parts = drv.split(sep) + if len(drv_parts) == 4 and drv_parts[2] not in '?.': + # e.g. //server/share + root = sep + elif len(drv_parts) == 6: + # e.g. //?/unc/server/share + root = sep + return drv, root, [x for x in rel.split(sep) if x and x != '.'] + + @classmethod + def _parse_pattern(cls, pattern): + """Parse a glob pattern to a list of parts. This is much like + _parse_path, except: + + - Rather than normalizing and returning the drive and root, we raise + NotImplementedError if either are present. + - If the path has no real parts, we raise ValueError. + - If the path ends in a slash, then a final empty part is added. + """ + drv, root, rel = cls.parser.splitroot(pattern) + if root or drv: + raise NotImplementedError("Non-relative patterns are unsupported") + sep = cls.parser.sep + altsep = cls.parser.altsep + if altsep: + rel = rel.replace(altsep, sep) + parts = [x for x in rel.split(sep) if x and x != '.'] + if not parts: + raise ValueError(f"Unacceptable pattern: {str(pattern)!r}") + elif rel.endswith(sep): + # GH-65238: preserve trailing slash in glob patterns. + parts.append('') + return parts + + def as_posix(self): + """Return the string representation of the path with forward (/) + slashes.""" + return str(self).replace(self.parser.sep, '/') + + @property + def _raw_path(self): + paths = self._raw_paths + if len(paths) == 1: + return paths[0] + elif paths: + # Join path segments from the initializer. + return self.parser.join(*paths) + else: + return '' + + @property + def drive(self): + """The drive prefix (letter or UNC path), if any.""" + try: + return self._drv + except AttributeError: + self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) + return self._drv + + @property + def root(self): + """The root of the path, if any.""" + try: + return self._root + except AttributeError: + self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) + return self._root + + @property + def _tail(self): + try: + return self._tail_cached + except AttributeError: + self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) + return self._tail_cached + + @property + def anchor(self): + """The concatenation of the drive and root, or ''.""" + return self.drive + self.root + + @property + def parts(self): + """An object providing sequence-like access to the + components in the filesystem path.""" + if self.drive or self.root: + return (self.drive + self.root,) + tuple(self._tail) + else: + return tuple(self._tail) + + @property + def parent(self): + """The logical parent of the path.""" + drv = self.drive + root = self.root + tail = self._tail + if not tail: + return self + return self._from_parsed_parts(drv, root, tail[:-1]) + + @property + def parents(self): + """A sequence of this path's logical parents.""" + # The value of this property should not be cached on the path object, + # as doing so would introduce a reference cycle. + return _PathParents(self) + + @property + def name(self): + """The final path component, if any.""" + tail = self._tail + if not tail: + return '' + return tail[-1] + + def with_name(self, name): + """Return a new path with the file name changed.""" + p = self.parser + if not name or p.sep in name or (p.altsep and p.altsep in name) or name == '.': + raise ValueError(f"Invalid name {name!r}") + tail = self._tail.copy() + if not tail: + raise ValueError(f"{self!r} has an empty name") + tail[-1] = name + return self._from_parsed_parts(self.drive, self.root, tail) + + def with_stem(self, stem): + """Return a new path with the stem changed.""" + suffix = self.suffix + if not suffix: + return self.with_name(stem) + elif not stem: + # If the suffix is non-empty, we can't make the stem empty. + raise ValueError(f"{self!r} has a non-empty suffix") + else: + return self.with_name(stem + suffix) + + def with_suffix(self, suffix): + """Return a new path with the file suffix changed. If the path + has no suffix, add given suffix. If the given suffix is an empty + string, remove the suffix from the path. + """ + stem = self.stem + if not stem: + # If the stem is empty, we can't make the suffix non-empty. + raise ValueError(f"{self!r} has an empty name") + elif suffix and not suffix.startswith('.'): + raise ValueError(f"Invalid suffix {suffix!r}") + else: + return self.with_name(stem + suffix) + + @property + def stem(self): + """The final path component, minus its last suffix.""" + name = self.name + i = name.rfind('.') + if i != -1: + stem = name[:i] + # Stem must contain at least one non-dot character. + if stem.lstrip('.'): + return stem + return name + + @property + def suffix(self): + """ + The final component's last suffix, if any. + + This includes the leading period. For example: '.txt' + """ + name = self.name.lstrip('.') + i = name.rfind('.') + if i != -1: + return name[i:] + return '' + + @property + def suffixes(self): + """ + A list of the final component's suffixes, if any. + + These include the leading periods. For example: ['.tar', '.gz'] + """ + return ['.' + ext for ext in self.name.lstrip('.').split('.')[1:]] + + def relative_to(self, other, *, walk_up=False): + """Return the relative path to another path identified by the passed + arguments. If the operation is not possible (because this is not + related to the other path), raise ValueError. + + The *walk_up* parameter controls whether `..` may be used to resolve + the path. + """ + if not hasattr(other, 'with_segments'): + other = self.with_segments(other) + for step, path in enumerate(chain([other], other.parents)): + if path == self or path in self.parents: + break + elif not walk_up: + raise ValueError(f"{str(self)!r} is not in the subpath of {str(other)!r}") + elif path.name == '..': + raise ValueError(f"'..' segment in {str(other)!r} cannot be walked") + else: + raise ValueError(f"{str(self)!r} and {str(other)!r} have different anchors") + parts = ['..'] * step + self._tail[len(path._tail):] + return self._from_parsed_parts('', '', parts) + + def is_relative_to(self, other): + """Return True if the path is relative to another path or False. + """ + if not hasattr(other, 'with_segments'): + other = self.with_segments(other) + return other == self or other in self.parents + + def is_absolute(self): + """True if the path is absolute (has both a root and, if applicable, + a drive).""" + if self.parser is posixpath: + # Optimization: work with raw paths on POSIX. + for path in self._raw_paths: + if path.startswith('/'): + return True + return False + return self.parser.isabs(self) + + def is_reserved(self): + """Return True if the path contains one of the special names reserved + by the system, if any.""" + import warnings + msg = ("pathlib.PurePath.is_reserved() is deprecated and scheduled " + "for removal in Python 3.15. Use os.path.isreserved() to " + "detect reserved paths on Windows.") + warnings._deprecated("pathlib.PurePath.is_reserved", msg, remove=(3, 15)) + if self.parser is ntpath: + return self.parser.isreserved(self) + return False + + def as_uri(self): + """Return the path as a URI.""" + import warnings + msg = ("pathlib.PurePath.as_uri() is deprecated and scheduled " + "for removal in Python 3.19. Use pathlib.Path.as_uri().") + warnings._deprecated("pathlib.PurePath.as_uri", msg, remove=(3, 19)) + if not self.is_absolute(): + raise ValueError("relative path can't be expressed as a file URI") + + drive = self.drive + if len(drive) == 2 and drive[1] == ':': + # It's a path on a local drive => 'file:///c:/a/b' + prefix = 'file:///' + drive + path = self.as_posix()[2:] + elif drive: + # It's a path on a network drive => 'file://host/share/a/b' + prefix = 'file:' + path = self.as_posix() + else: + # It's a posix path => 'file:///etc/hosts' + prefix = 'file://' + path = str(self) + from urllib.parse import quote_from_bytes + return prefix + quote_from_bytes(os.fsencode(path)) + + def full_match(self, pattern, *, case_sensitive=None): + """ + Return True if this path matches the given glob-style pattern. The + pattern is matched against the entire path. + """ + if not hasattr(pattern, 'with_segments'): + pattern = self.with_segments(pattern) + if case_sensitive is None: + case_sensitive = self.parser is posixpath + + # The string representation of an empty path is a single dot ('.'). Empty + # paths shouldn't match wildcards, so we change it to the empty string. + path = str(self) if self.parts else '' + pattern = str(pattern) if pattern.parts else '' + globber = _StringGlobber(self.parser.sep, case_sensitive, recursive=True) + return globber.compile(pattern)(path) is not None + + def match(self, path_pattern, *, case_sensitive=None): + """ + Return True if this path matches the given pattern. If the pattern is + relative, matching is done from the right; otherwise, the entire path + is matched. The recursive wildcard '**' is *not* supported by this + method. + """ + if not hasattr(path_pattern, 'with_segments'): + path_pattern = self.with_segments(path_pattern) + if case_sensitive is None: + case_sensitive = self.parser is posixpath + path_parts = self.parts[::-1] + pattern_parts = path_pattern.parts[::-1] + if not pattern_parts: + raise ValueError("empty pattern") + if len(path_parts) < len(pattern_parts): + return False + if len(path_parts) > len(pattern_parts) and path_pattern.anchor: + return False + globber = _StringGlobber(self.parser.sep, case_sensitive) + for path_part, pattern_part in zip(path_parts, pattern_parts): + match = globber.compile(pattern_part) + if match(path_part) is None: + return False + return True + +# Subclassing os.PathLike makes isinstance() checks slower, +# which in turn makes Path construction slower. Register instead! +os.PathLike.register(PurePath) + + +class PurePosixPath(PurePath): + """PurePath subclass for non-Windows systems. + + On a POSIX system, instantiating a PurePath should return this object. + However, you can also instantiate it directly on any system. + """ + parser = posixpath + __slots__ = () + + +class PureWindowsPath(PurePath): + """PurePath subclass for Windows systems. + + On a Windows system, instantiating a PurePath should return this object. + However, you can also instantiate it directly on any system. + """ + parser = ntpath + __slots__ = () + + +class Path(PurePath): + """PurePath subclass that can make system calls. + + Path represents a filesystem path but unlike PurePath, also offers + methods to do system calls on path objects. Depending on your system, + instantiating a Path will return either a PosixPath or a WindowsPath + object. You can also instantiate a PosixPath or WindowsPath directly, + but cannot instantiate a WindowsPath on a POSIX system or vice versa. + """ + __slots__ = ('_info',) + + def __new__(cls, *args, **kwargs): + if cls is Path: + cls = WindowsPath if os.name == 'nt' else PosixPath + return object.__new__(cls) + + @property + def info(self): + """ + A PathInfo object that exposes the file type and other file attributes + of this path. + """ + try: + return self._info + except AttributeError: + self._info = PathInfo(self) + return self._info + + def stat(self, *, follow_symlinks=True): + """ + Return the result of the stat() system call on this path, like + os.stat() does. + """ + return os.stat(self, follow_symlinks=follow_symlinks) + + def lstat(self): + """ + Like stat(), except if the path points to a symlink, the symlink's + status information is returned, rather than its target's. + """ + return os.lstat(self) + + def exists(self, *, follow_symlinks=True): + """ + Whether this path exists. + + This method normally follows symlinks; to check whether a symlink exists, + add the argument follow_symlinks=False. + """ + if follow_symlinks: + return os.path.exists(self) + return os.path.lexists(self) + + def is_dir(self, *, follow_symlinks=True): + """ + Whether this path is a directory. + """ + if follow_symlinks: + return os.path.isdir(self) + try: + return S_ISDIR(self.stat(follow_symlinks=follow_symlinks).st_mode) + except (OSError, ValueError): + return False + + def is_file(self, *, follow_symlinks=True): + """ + Whether this path is a regular file (also True for symlinks pointing + to regular files). + """ + if follow_symlinks: + return os.path.isfile(self) + try: + return S_ISREG(self.stat(follow_symlinks=follow_symlinks).st_mode) + except (OSError, ValueError): + return False + + def is_mount(self): + """ + Check if this path is a mount point + """ + return os.path.ismount(self) + + def is_symlink(self): + """ + Whether this path is a symbolic link. + """ + return os.path.islink(self) + + def is_junction(self): + """ + Whether this path is a junction. + """ + return os.path.isjunction(self) + + def is_block_device(self): + """ + Whether this path is a block device. + """ + try: + return S_ISBLK(self.stat().st_mode) + except (OSError, ValueError): + return False + + def is_char_device(self): + """ + Whether this path is a character device. + """ + try: + return S_ISCHR(self.stat().st_mode) + except (OSError, ValueError): + return False + + def is_fifo(self): + """ + Whether this path is a FIFO. + """ + try: + return S_ISFIFO(self.stat().st_mode) + except (OSError, ValueError): + return False + + def is_socket(self): + """ + Whether this path is a socket. + """ + try: + return S_ISSOCK(self.stat().st_mode) + except (OSError, ValueError): + return False + + def samefile(self, other_path): + """Return whether other_path is the same or not as this file + (as returned by os.path.samefile()). + """ + st = self.stat() + try: + other_st = other_path.stat() + except AttributeError: + other_st = self.with_segments(other_path).stat() + return (st.st_ino == other_st.st_ino and + st.st_dev == other_st.st_dev) + + def open(self, mode='r', buffering=-1, encoding=None, + errors=None, newline=None): + """ + Open the file pointed to by this path and return a file object, as + the built-in open() function does. + """ + if "b" not in mode: + encoding = io.text_encoding(encoding) + return io.open(self, mode, buffering, encoding, errors, newline) + + def read_bytes(self): + """ + Open the file in bytes mode, read it, and close the file. + """ + with self.open(mode='rb', buffering=0) as f: + return f.read() + + def read_text(self, encoding=None, errors=None, newline=None): + """ + Open the file in text mode, read it, and close the file. + """ + # Call io.text_encoding() here to ensure any warning is raised at an + # appropriate stack level. + encoding = io.text_encoding(encoding) + with self.open(mode='r', encoding=encoding, errors=errors, newline=newline) as f: + return f.read() + + def write_bytes(self, data): + """ + Open the file in bytes mode, write to it, and close the file. + """ + # type-check for the buffer interface before truncating the file + view = memoryview(data) + with self.open(mode='wb') as f: + return f.write(view) + + def write_text(self, data, encoding=None, errors=None, newline=None): + """ + Open the file in text mode, write to it, and close the file. + """ + # Call io.text_encoding() here to ensure any warning is raised at an + # appropriate stack level. + encoding = io.text_encoding(encoding) + if not isinstance(data, str): + raise TypeError('data must be str, not %s' % + data.__class__.__name__) + with self.open(mode='w', encoding=encoding, errors=errors, newline=newline) as f: + return f.write(data) + + _remove_leading_dot = operator.itemgetter(slice(2, None)) + _remove_trailing_slash = operator.itemgetter(slice(-1)) + + def _filter_trailing_slash(self, paths): + sep = self.parser.sep + anchor_len = len(self.anchor) + for path_str in paths: + if len(path_str) > anchor_len and path_str[-1] == sep: + path_str = path_str[:-1] + yield path_str + + def _from_dir_entry(self, dir_entry, path_str): + path = self.with_segments(path_str) + path._str = path_str + path._info = DirEntryInfo(dir_entry) + return path + + def iterdir(self): + """Yield path objects of the directory contents. + + The children are yielded in arbitrary order, and the + special entries '.' and '..' are not included. + """ + root_dir = str(self) + with os.scandir(root_dir) as scandir_it: + entries = list(scandir_it) + if root_dir == '.': + return (self._from_dir_entry(e, e.name) for e in entries) + else: + return (self._from_dir_entry(e, e.path) for e in entries) + + def glob(self, pattern, *, case_sensitive=None, recurse_symlinks=False): + """Iterate over this subtree and yield all existing files (of any + kind, including directories) matching the given relative pattern. + """ + sys.audit("pathlib.Path.glob", self, pattern) + if case_sensitive is None: + case_sensitive = self.parser is posixpath + case_pedantic = False + else: + # The user has expressed a case sensitivity choice, but we don't + # know the case sensitivity of the underlying filesystem, so we + # must use scandir() for everything, including non-wildcard parts. + case_pedantic = True + parts = self._parse_pattern(pattern) + recursive = True if recurse_symlinks else _no_recurse_symlinks + globber = _StringGlobber(self.parser.sep, case_sensitive, case_pedantic, recursive) + select = globber.selector(parts[::-1]) + root = str(self) + paths = select(self.parser.join(root, '')) + + # Normalize results + if root == '.': + paths = map(self._remove_leading_dot, paths) + if parts[-1] == '': + paths = map(self._remove_trailing_slash, paths) + elif parts[-1] == '**': + paths = self._filter_trailing_slash(paths) + paths = map(self._from_parsed_string, paths) + return paths + + def rglob(self, pattern, *, case_sensitive=None, recurse_symlinks=False): + """Recursively yield all existing files (of any kind, including + directories) matching the given relative pattern, anywhere in + this subtree. + """ + sys.audit("pathlib.Path.rglob", self, pattern) + pattern = self.parser.join('**', pattern) + return self.glob(pattern, case_sensitive=case_sensitive, recurse_symlinks=recurse_symlinks) + + def walk(self, top_down=True, on_error=None, follow_symlinks=False): + """Walk the directory tree from this directory, similar to os.walk().""" + sys.audit("pathlib.Path.walk", self, on_error, follow_symlinks) + root_dir = str(self) + if not follow_symlinks: + follow_symlinks = os._walk_symlinks_as_files + results = os.walk(root_dir, top_down, on_error, follow_symlinks) + for path_str, dirnames, filenames in results: + if root_dir == '.': + path_str = path_str[2:] + yield self._from_parsed_string(path_str), dirnames, filenames + + def absolute(self): + """Return an absolute version of this path + No normalization or symlink resolution is performed. + + Use resolve() to resolve symlinks and remove '..' segments. + """ + if self.is_absolute(): + return self + if self.root: + drive = os.path.splitroot(os.getcwd())[0] + return self._from_parsed_parts(drive, self.root, self._tail) + if self.drive: + # There is a CWD on each drive-letter drive. + cwd = os.path.abspath(self.drive) + else: + cwd = os.getcwd() + if not self._tail: + # Fast path for "empty" paths, e.g. Path("."), Path("") or Path(). + # We pass only one argument to with_segments() to avoid the cost + # of joining, and we exploit the fact that getcwd() returns a + # fully-normalized string by storing it in _str. This is used to + # implement Path.cwd(). + return self._from_parsed_string(cwd) + drive, root, rel = os.path.splitroot(cwd) + if not rel: + return self._from_parsed_parts(drive, root, self._tail) + tail = rel.split(self.parser.sep) + tail.extend(self._tail) + return self._from_parsed_parts(drive, root, tail) + + @classmethod + def cwd(cls): + """Return a new path pointing to the current working directory.""" + cwd = os.getcwd() + path = cls(cwd) + path._str = cwd # getcwd() returns a normalized path + return path + + def resolve(self, strict=False): + """ + Make the path absolute, resolving all symlinks on the way and also + normalizing it. + """ + + return self.with_segments(os.path.realpath(self, strict=strict)) + + if pwd: + def owner(self, *, follow_symlinks=True): + """ + Return the login name of the file owner. + """ + uid = self.stat(follow_symlinks=follow_symlinks).st_uid + return pwd.getpwuid(uid).pw_name + else: + def owner(self, *, follow_symlinks=True): + """ + Return the login name of the file owner. + """ + f = f"{type(self).__name__}.owner()" + raise UnsupportedOperation(f"{f} is unsupported on this system") + + if grp: + def group(self, *, follow_symlinks=True): + """ + Return the group name of the file gid. + """ + gid = self.stat(follow_symlinks=follow_symlinks).st_gid + return grp.getgrgid(gid).gr_name + else: + def group(self, *, follow_symlinks=True): + """ + Return the group name of the file gid. + """ + f = f"{type(self).__name__}.group()" + raise UnsupportedOperation(f"{f} is unsupported on this system") + + if hasattr(os, "readlink"): + def readlink(self): + """ + Return the path to which the symbolic link points. + """ + return self.with_segments(os.readlink(self)) + else: + def readlink(self): + """ + Return the path to which the symbolic link points. + """ + f = f"{type(self).__name__}.readlink()" + raise UnsupportedOperation(f"{f} is unsupported on this system") + + def touch(self, mode=0o666, exist_ok=True): + """ + Create this file with the given access mode, if it doesn't exist. + """ + + if exist_ok: + # First try to bump modification time + # Implementation note: GNU touch uses the UTIME_NOW option of + # the utimensat() / futimens() functions. + try: + os.utime(self, None) + except OSError: + # Avoid exception chaining + pass + else: + return + flags = os.O_CREAT | os.O_WRONLY + if not exist_ok: + flags |= os.O_EXCL + fd = os.open(self, flags, mode) + os.close(fd) + + def mkdir(self, mode=0o777, parents=False, exist_ok=False): + """ + Create a new directory at this given path. + """ + try: + os.mkdir(self, mode) + except FileNotFoundError: + if not parents or self.parent == self: + raise + self.parent.mkdir(parents=True, exist_ok=True) + self.mkdir(mode, parents=False, exist_ok=exist_ok) + except OSError: + # Cannot rely on checking for EEXIST, since the operating system + # could give priority to other errors like EACCES or EROFS + if not exist_ok or not self.is_dir(): + raise + + def chmod(self, mode, *, follow_symlinks=True): + """ + Change the permissions of the path, like os.chmod(). + """ + os.chmod(self, mode, follow_symlinks=follow_symlinks) + + def lchmod(self, mode): + """ + Like chmod(), except if the path points to a symlink, the symlink's + permissions are changed, rather than its target's. + """ + self.chmod(mode, follow_symlinks=False) + + def unlink(self, missing_ok=False): + """ + Remove this file or link. + If the path is a directory, use rmdir() instead. + """ + try: + os.unlink(self) + except FileNotFoundError: + if not missing_ok: + raise + + def rmdir(self): + """ + Remove this directory. The directory must be empty. + """ + os.rmdir(self) + + def _delete(self): + """ + Delete this file or directory (including all sub-directories). + """ + if self.is_symlink() or self.is_junction(): + self.unlink() + elif self.is_dir(): + # Lazy import to improve module import time + import shutil + shutil.rmtree(self) + else: + self.unlink() + + def rename(self, target): + """ + Rename this path to the target path. + + The target path may be absolute or relative. Relative paths are + interpreted relative to the current working directory, *not* the + directory of the Path object. + + Returns the new Path instance pointing to the target path. + """ + os.rename(self, target) + if not hasattr(target, 'with_segments'): + target = self.with_segments(target) + return target + + def replace(self, target): + """ + Rename this path to the target path, overwriting if that path exists. + + The target path may be absolute or relative. Relative paths are + interpreted relative to the current working directory, *not* the + directory of the Path object. + + Returns the new Path instance pointing to the target path. + """ + os.replace(self, target) + if not hasattr(target, 'with_segments'): + target = self.with_segments(target) + return target + + def copy(self, target, **kwargs): + """ + Recursively copy this file or directory tree to the given destination. + """ + if not hasattr(target, 'with_segments'): + target = self.with_segments(target) + ensure_distinct_paths(self, target) + target._copy_from(self, **kwargs) + return target.joinpath() # Empty join to ensure fresh metadata. + + def copy_into(self, target_dir, **kwargs): + """ + Copy this file or directory tree into the given existing directory. + """ + name = self.name + if not name: + raise ValueError(f"{self!r} has an empty name") + elif hasattr(target_dir, 'with_segments'): + target = target_dir / name + else: + target = self.with_segments(target_dir, name) + return self.copy(target, **kwargs) + + def _copy_from(self, source, follow_symlinks=True, preserve_metadata=False): + """ + Recursively copy the given path to this path. + """ + if not follow_symlinks and source.info.is_symlink(): + self._copy_from_symlink(source, preserve_metadata) + elif source.info.is_dir(): + children = source.iterdir() + os.mkdir(self) + for child in children: + self.joinpath(child.name)._copy_from( + child, follow_symlinks, preserve_metadata) + if preserve_metadata: + copy_info(source.info, self) + else: + self._copy_from_file(source, preserve_metadata) + + def _copy_from_file(self, source, preserve_metadata=False): + ensure_different_files(source, self) + with magic_open(source, 'rb') as source_f: + with open(self, 'wb') as target_f: + copyfileobj(source_f, target_f) + if preserve_metadata: + copy_info(source.info, self) + + if copyfile2: + # Use fast OS routine for local file copying where available. + _copy_from_file_fallback = _copy_from_file + def _copy_from_file(self, source, preserve_metadata=False): + try: + source = os.fspath(source) + except TypeError: + pass + else: + copyfile2(source, str(self)) + return + self._copy_from_file_fallback(source, preserve_metadata) + + if os.name == 'nt': + # If a directory-symlink is copied *before* its target, then + # os.symlink() incorrectly creates a file-symlink on Windows. Avoid + # this by passing *target_is_dir* to os.symlink() on Windows. + def _copy_from_symlink(self, source, preserve_metadata=False): + os.symlink(str(source.readlink()), self, source.info.is_dir()) + if preserve_metadata: + copy_info(source.info, self, follow_symlinks=False) + else: + def _copy_from_symlink(self, source, preserve_metadata=False): + os.symlink(str(source.readlink()), self) + if preserve_metadata: + copy_info(source.info, self, follow_symlinks=False) + + def move(self, target): + """ + Recursively move this file or directory tree to the given destination. + """ + # Use os.replace() if the target is os.PathLike and on the same FS. + try: + target = self.with_segments(target) + except TypeError: + pass + else: + ensure_different_files(self, target) + try: + os.replace(self, target) + except OSError as err: + if err.errno != EXDEV: + raise + else: + return target.joinpath() # Empty join to ensure fresh metadata. + # Fall back to copy+delete. + target = self.copy(target, follow_symlinks=False, preserve_metadata=True) + self._delete() + return target + + def move_into(self, target_dir): + """ + Move this file or directory tree into the given existing directory. + """ + name = self.name + if not name: + raise ValueError(f"{self!r} has an empty name") + elif hasattr(target_dir, 'with_segments'): + target = target_dir / name + else: + target = self.with_segments(target_dir, name) + return self.move(target) + + if hasattr(os, "symlink"): + def symlink_to(self, target, target_is_directory=False): + """ + Make this path a symlink pointing to the target path. + Note the order of arguments (link, target) is the reverse of os.symlink. + """ + os.symlink(target, self, target_is_directory) + else: + def symlink_to(self, target, target_is_directory=False): + """ + Make this path a symlink pointing to the target path. + Note the order of arguments (link, target) is the reverse of os.symlink. + """ + f = f"{type(self).__name__}.symlink_to()" + raise UnsupportedOperation(f"{f} is unsupported on this system") + + if hasattr(os, "link"): + def hardlink_to(self, target): + """ + Make this path a hard link pointing to the same file as *target*. + + Note the order of arguments (self, target) is the reverse of os.link's. + """ + os.link(target, self) + else: + def hardlink_to(self, target): + """ + Make this path a hard link pointing to the same file as *target*. + + Note the order of arguments (self, target) is the reverse of os.link's. + """ + f = f"{type(self).__name__}.hardlink_to()" + raise UnsupportedOperation(f"{f} is unsupported on this system") + + def expanduser(self): + """ Return a new path with expanded ~ and ~user constructs + (as returned by os.path.expanduser) + """ + if (not (self.drive or self.root) and + self._tail and self._tail[0][:1] == '~'): + homedir = os.path.expanduser(self._tail[0]) + if homedir[:1] == "~": + raise RuntimeError("Could not determine home directory.") + drv, root, tail = self._parse_path(homedir) + return self._from_parsed_parts(drv, root, tail + self._tail[1:]) + + return self + + @classmethod + def home(cls): + """Return a new path pointing to expanduser('~'). + """ + homedir = os.path.expanduser("~") + if homedir == "~": + raise RuntimeError("Could not determine home directory.") + return cls(homedir) + + def as_uri(self): + """Return the path as a URI.""" + if not self.is_absolute(): + raise ValueError("relative paths can't be expressed as file URIs") + from urllib.request import pathname2url + return pathname2url(str(self), add_scheme=True) + + @classmethod + def from_uri(cls, uri): + """Return a new path from the given 'file' URI.""" + from urllib.error import URLError + from urllib.request import url2pathname + try: + path = cls(url2pathname(uri, require_scheme=True)) + except URLError as exc: + raise ValueError(exc.reason) from None + if not path.is_absolute(): + raise ValueError(f"URI is not absolute: {uri!r}") + return path + + +class PosixPath(Path, PurePosixPath): + """Path subclass for non-Windows systems. + + On a POSIX system, instantiating a Path should return this object. + """ + __slots__ = () + + if os.name == 'nt': + def __new__(cls, *args, **kwargs): + raise UnsupportedOperation( + f"cannot instantiate {cls.__name__!r} on your system") + +class WindowsPath(Path, PureWindowsPath): + """Path subclass for Windows systems. + + On a Windows system, instantiating a Path should return this object. + """ + __slots__ = () + + if os.name != 'nt': + def __new__(cls, *args, **kwargs): + raise UnsupportedOperation( + f"cannot instantiate {cls.__name__!r} on your system") diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py deleted file mode 100644 index 4d24146aa53..00000000000 --- a/Lib/pathlib/_abc.py +++ /dev/null @@ -1,930 +0,0 @@ -""" -Abstract base classes for rich path objects. - -This module is published as a PyPI package called "pathlib-abc". - -This module is also a *PRIVATE* part of the Python standard library, where -it's developed alongside pathlib. If it finds success and maturity as a PyPI -package, it could become a public part of the standard library. - -Two base classes are defined here -- PurePathBase and PathBase -- that -resemble pathlib's PurePath and Path respectively. -""" - -import functools -from glob import _Globber, _no_recurse_symlinks -from errno import ENOENT, ENOTDIR, EBADF, ELOOP, EINVAL -from stat import S_ISDIR, S_ISLNK, S_ISREG, S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO - - -__all__ = ["UnsupportedOperation"] - -# -# Internals -# - -_WINERROR_NOT_READY = 21 # drive exists but is not accessible -_WINERROR_INVALID_NAME = 123 # fix for bpo-35306 -_WINERROR_CANT_RESOLVE_FILENAME = 1921 # broken symlink pointing to itself - -# EBADF - guard against macOS `stat` throwing EBADF -_IGNORED_ERRNOS = (ENOENT, ENOTDIR, EBADF, ELOOP) - -_IGNORED_WINERRORS = ( - _WINERROR_NOT_READY, - _WINERROR_INVALID_NAME, - _WINERROR_CANT_RESOLVE_FILENAME) - -def _ignore_error(exception): - return (getattr(exception, 'errno', None) in _IGNORED_ERRNOS or - getattr(exception, 'winerror', None) in _IGNORED_WINERRORS) - - -@functools.cache -def _is_case_sensitive(parser): - return parser.normcase('Aa') == 'Aa' - - -class UnsupportedOperation(NotImplementedError): - """An exception that is raised when an unsupported operation is called on - a path object. - """ - pass - - -class ParserBase: - """Base class for path parsers, which do low-level path manipulation. - - Path parsers provide a subset of the os.path API, specifically those - functions needed to provide PurePathBase functionality. Each PurePathBase - subclass references its path parser via a 'parser' class attribute. - - Every method in this base class raises an UnsupportedOperation exception. - """ - - @classmethod - def _unsupported_msg(cls, attribute): - return f"{cls.__name__}.{attribute} is unsupported" - - @property - def sep(self): - """The character used to separate path components.""" - raise UnsupportedOperation(self._unsupported_msg('sep')) - - def join(self, path, *paths): - """Join path segments.""" - raise UnsupportedOperation(self._unsupported_msg('join()')) - - def split(self, path): - """Split the path into a pair (head, tail), where *head* is everything - before the final path separator, and *tail* is everything after. - Either part may be empty. - """ - raise UnsupportedOperation(self._unsupported_msg('split()')) - - def splitdrive(self, path): - """Split the path into a 2-item tuple (drive, tail), where *drive* is - a device name or mount point, and *tail* is everything after the - drive. Either part may be empty.""" - raise UnsupportedOperation(self._unsupported_msg('splitdrive()')) - - def normcase(self, path): - """Normalize the case of the path.""" - raise UnsupportedOperation(self._unsupported_msg('normcase()')) - - def isabs(self, path): - """Returns whether the path is absolute, i.e. unaffected by the - current directory or drive.""" - raise UnsupportedOperation(self._unsupported_msg('isabs()')) - - -class PurePathBase: - """Base class for pure path objects. - - This class *does not* provide several magic methods that are defined in - its subclass PurePath. They are: __fspath__, __bytes__, __reduce__, - __hash__, __eq__, __lt__, __le__, __gt__, __ge__. Its initializer and path - joining methods accept only strings, not os.PathLike objects more broadly. - """ - - __slots__ = ( - # The `_raw_path` slot store a joined string path. This is set in the - # `__init__()` method. - '_raw_path', - - # The '_resolving' slot stores a boolean indicating whether the path - # is being processed by `PathBase.resolve()`. This prevents duplicate - # work from occurring when `resolve()` calls `stat()` or `readlink()`. - '_resolving', - ) - parser = ParserBase() - _globber = _Globber - - def __init__(self, path, *paths): - self._raw_path = self.parser.join(path, *paths) if paths else path - if not isinstance(self._raw_path, str): - raise TypeError( - f"path should be a str, not {type(self._raw_path).__name__!r}") - self._resolving = False - - def with_segments(self, *pathsegments): - """Construct a new path object from any number of path-like objects. - Subclasses may override this method to customize how new path objects - are created from methods like `iterdir()`. - """ - return type(self)(*pathsegments) - - def __str__(self): - """Return the string representation of the path, suitable for - passing to system calls.""" - return self._raw_path - - def as_posix(self): - """Return the string representation of the path with forward (/) - slashes.""" - return str(self).replace(self.parser.sep, '/') - - @property - def drive(self): - """The drive prefix (letter or UNC path), if any.""" - return self.parser.splitdrive(self.anchor)[0] - - @property - def root(self): - """The root of the path, if any.""" - return self.parser.splitdrive(self.anchor)[1] - - @property - def anchor(self): - """The concatenation of the drive and root, or ''.""" - return self._stack[0] - - @property - def name(self): - """The final path component, if any.""" - return self.parser.split(self._raw_path)[1] - - @property - def suffix(self): - """ - The final component's last suffix, if any. - - This includes the leading period. For example: '.txt' - """ - name = self.name - i = name.rfind('.') - if 0 < i < len(name) - 1: - return name[i:] - else: - return '' - - @property - def suffixes(self): - """ - A list of the final component's suffixes, if any. - - These include the leading periods. For example: ['.tar', '.gz'] - """ - name = self.name - if name.endswith('.'): - return [] - name = name.lstrip('.') - return ['.' + suffix for suffix in name.split('.')[1:]] - - @property - def stem(self): - """The final path component, minus its last suffix.""" - name = self.name - i = name.rfind('.') - if 0 < i < len(name) - 1: - return name[:i] - else: - return name - - def with_name(self, name): - """Return a new path with the file name changed.""" - split = self.parser.split - if split(name)[0]: - raise ValueError(f"Invalid name {name!r}") - return self.with_segments(split(self._raw_path)[0], name) - - def with_stem(self, stem): - """Return a new path with the stem changed.""" - suffix = self.suffix - if not suffix: - return self.with_name(stem) - elif not stem: - # If the suffix is non-empty, we can't make the stem empty. - raise ValueError(f"{self!r} has a non-empty suffix") - else: - return self.with_name(stem + suffix) - - def with_suffix(self, suffix): - """Return a new path with the file suffix changed. If the path - has no suffix, add given suffix. If the given suffix is an empty - string, remove the suffix from the path. - """ - stem = self.stem - if not stem: - # If the stem is empty, we can't make the suffix non-empty. - raise ValueError(f"{self!r} has an empty name") - elif suffix and not (suffix.startswith('.') and len(suffix) > 1): - raise ValueError(f"Invalid suffix {suffix!r}") - else: - return self.with_name(stem + suffix) - - def relative_to(self, other, *, walk_up=False): - """Return the relative path to another path identified by the passed - arguments. If the operation is not possible (because this is not - related to the other path), raise ValueError. - - The *walk_up* parameter controls whether `..` may be used to resolve - the path. - """ - if not isinstance(other, PurePathBase): - other = self.with_segments(other) - anchor0, parts0 = self._stack - anchor1, parts1 = other._stack - if anchor0 != anchor1: - raise ValueError(f"{self._raw_path!r} and {other._raw_path!r} have different anchors") - while parts0 and parts1 and parts0[-1] == parts1[-1]: - parts0.pop() - parts1.pop() - for part in parts1: - if not part or part == '.': - pass - elif not walk_up: - raise ValueError(f"{self._raw_path!r} is not in the subpath of {other._raw_path!r}") - elif part == '..': - raise ValueError(f"'..' segment in {other._raw_path!r} cannot be walked") - else: - parts0.append('..') - return self.with_segments('', *reversed(parts0)) - - def is_relative_to(self, other): - """Return True if the path is relative to another path or False. - """ - if not isinstance(other, PurePathBase): - other = self.with_segments(other) - anchor0, parts0 = self._stack - anchor1, parts1 = other._stack - if anchor0 != anchor1: - return False - while parts0 and parts1 and parts0[-1] == parts1[-1]: - parts0.pop() - parts1.pop() - for part in parts1: - if part and part != '.': - return False - return True - - @property - def parts(self): - """An object providing sequence-like access to the - components in the filesystem path.""" - anchor, parts = self._stack - if anchor: - parts.append(anchor) - return tuple(reversed(parts)) - - def joinpath(self, *pathsegments): - """Combine this path with one or several arguments, and return a - new path representing either a subpath (if all arguments are relative - paths) or a totally different path (if one of the arguments is - anchored). - """ - return self.with_segments(self._raw_path, *pathsegments) - - def __truediv__(self, key): - try: - return self.with_segments(self._raw_path, key) - except TypeError: - return NotImplemented - - def __rtruediv__(self, key): - try: - return self.with_segments(key, self._raw_path) - except TypeError: - return NotImplemented - - @property - def _stack(self): - """ - Split the path into a 2-tuple (anchor, parts), where *anchor* is the - uppermost parent of the path (equivalent to path.parents[-1]), and - *parts* is a reversed list of parts following the anchor. - """ - split = self.parser.split - path = self._raw_path - parent, name = split(path) - names = [] - while path != parent: - names.append(name) - path = parent - parent, name = split(path) - return path, names - - @property - def parent(self): - """The logical parent of the path.""" - path = self._raw_path - parent = self.parser.split(path)[0] - if path != parent: - parent = self.with_segments(parent) - parent._resolving = self._resolving - return parent - return self - - @property - def parents(self): - """A sequence of this path's logical parents.""" - split = self.parser.split - path = self._raw_path - parent = split(path)[0] - parents = [] - while path != parent: - parents.append(self.with_segments(parent)) - path = parent - parent = split(path)[0] - return tuple(parents) - - def is_absolute(self): - """True if the path is absolute (has both a root and, if applicable, - a drive).""" - return self.parser.isabs(self._raw_path) - - @property - def _pattern_str(self): - """The path expressed as a string, for use in pattern-matching.""" - return str(self) - - def match(self, path_pattern, *, case_sensitive=None): - """ - Return True if this path matches the given pattern. If the pattern is - relative, matching is done from the right; otherwise, the entire path - is matched. The recursive wildcard '**' is *not* supported by this - method. - """ - if not isinstance(path_pattern, PurePathBase): - path_pattern = self.with_segments(path_pattern) - if case_sensitive is None: - case_sensitive = _is_case_sensitive(self.parser) - sep = path_pattern.parser.sep - path_parts = self.parts[::-1] - pattern_parts = path_pattern.parts[::-1] - if not pattern_parts: - raise ValueError("empty pattern") - if len(path_parts) < len(pattern_parts): - return False - if len(path_parts) > len(pattern_parts) and path_pattern.anchor: - return False - globber = self._globber(sep, case_sensitive) - for path_part, pattern_part in zip(path_parts, pattern_parts): - match = globber.compile(pattern_part) - if match(path_part) is None: - return False - return True - - def full_match(self, pattern, *, case_sensitive=None): - """ - Return True if this path matches the given glob-style pattern. The - pattern is matched against the entire path. - """ - if not isinstance(pattern, PurePathBase): - pattern = self.with_segments(pattern) - if case_sensitive is None: - case_sensitive = _is_case_sensitive(self.parser) - globber = self._globber(pattern.parser.sep, case_sensitive, recursive=True) - match = globber.compile(pattern._pattern_str) - return match(self._pattern_str) is not None - - - -class PathBase(PurePathBase): - """Base class for concrete path objects. - - This class provides dummy implementations for many methods that derived - classes can override selectively; the default implementations raise - UnsupportedOperation. The most basic methods, such as stat() and open(), - directly raise UnsupportedOperation; these basic methods are called by - other methods such as is_dir() and read_text(). - - The Path class derives this class to implement local filesystem paths. - Users may derive their own classes to implement virtual filesystem paths, - such as paths in archive files or on remote storage systems. - """ - __slots__ = () - - # Maximum number of symlinks to follow in resolve() - _max_symlinks = 40 - - @classmethod - def _unsupported_msg(cls, attribute): - return f"{cls.__name__}.{attribute} is unsupported" - - def stat(self, *, follow_symlinks=True): - """ - Return the result of the stat() system call on this path, like - os.stat() does. - """ - raise UnsupportedOperation(self._unsupported_msg('stat()')) - - def lstat(self): - """ - Like stat(), except if the path points to a symlink, the symlink's - status information is returned, rather than its target's. - """ - return self.stat(follow_symlinks=False) - - - # Convenience functions for querying the stat results - - def exists(self, *, follow_symlinks=True): - """ - Whether this path exists. - - This method normally follows symlinks; to check whether a symlink exists, - add the argument follow_symlinks=False. - """ - try: - self.stat(follow_symlinks=follow_symlinks) - except OSError as e: - if not _ignore_error(e): - raise - return False - except ValueError: - # Non-encodable path - return False - return True - - def is_dir(self, *, follow_symlinks=True): - """ - Whether this path is a directory. - """ - try: - return S_ISDIR(self.stat(follow_symlinks=follow_symlinks).st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - - def is_file(self, *, follow_symlinks=True): - """ - Whether this path is a regular file (also True for symlinks pointing - to regular files). - """ - try: - return S_ISREG(self.stat(follow_symlinks=follow_symlinks).st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - - def is_mount(self): - """ - Check if this path is a mount point - """ - # Need to exist and be a dir - if not self.exists() or not self.is_dir(): - return False - - try: - parent_dev = self.parent.stat().st_dev - except OSError: - return False - - dev = self.stat().st_dev - if dev != parent_dev: - return True - ino = self.stat().st_ino - parent_ino = self.parent.stat().st_ino - return ino == parent_ino - - def is_symlink(self): - """ - Whether this path is a symbolic link. - """ - try: - return S_ISLNK(self.lstat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist - return False - except ValueError: - # Non-encodable path - return False - - def is_junction(self): - """ - Whether this path is a junction. - """ - # Junctions are a Windows-only feature, not present in POSIX nor the - # majority of virtual filesystems. There is no cross-platform idiom - # to check for junctions (using stat().st_mode). - return False - - def is_block_device(self): - """ - Whether this path is a block device. - """ - try: - return S_ISBLK(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - - def is_char_device(self): - """ - Whether this path is a character device. - """ - try: - return S_ISCHR(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - - def is_fifo(self): - """ - Whether this path is a FIFO. - """ - try: - return S_ISFIFO(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - - def is_socket(self): - """ - Whether this path is a socket. - """ - try: - return S_ISSOCK(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - - def samefile(self, other_path): - """Return whether other_path is the same or not as this file - (as returned by os.path.samefile()). - """ - st = self.stat() - try: - other_st = other_path.stat() - except AttributeError: - other_st = self.with_segments(other_path).stat() - return (st.st_ino == other_st.st_ino and - st.st_dev == other_st.st_dev) - - def open(self, mode='r', buffering=-1, encoding=None, - errors=None, newline=None): - """ - Open the file pointed to by this path and return a file object, as - the built-in open() function does. - """ - raise UnsupportedOperation(self._unsupported_msg('open()')) - - def read_bytes(self): - """ - Open the file in bytes mode, read it, and close the file. - """ - with self.open(mode='rb') as f: - return f.read() - - def read_text(self, encoding=None, errors=None, newline=None): - """ - Open the file in text mode, read it, and close the file. - """ - with self.open(mode='r', encoding=encoding, errors=errors, newline=newline) as f: - return f.read() - - def write_bytes(self, data): - """ - Open the file in bytes mode, write to it, and close the file. - """ - # type-check for the buffer interface before truncating the file - view = memoryview(data) - with self.open(mode='wb') as f: - return f.write(view) - - def write_text(self, data, encoding=None, errors=None, newline=None): - """ - Open the file in text mode, write to it, and close the file. - """ - if not isinstance(data, str): - raise TypeError('data must be str, not %s' % - data.__class__.__name__) - with self.open(mode='w', encoding=encoding, errors=errors, newline=newline) as f: - return f.write(data) - - def iterdir(self): - """Yield path objects of the directory contents. - - The children are yielded in arbitrary order, and the - special entries '.' and '..' are not included. - """ - raise UnsupportedOperation(self._unsupported_msg('iterdir()')) - - def _glob_selector(self, parts, case_sensitive, recurse_symlinks): - if case_sensitive is None: - case_sensitive = _is_case_sensitive(self.parser) - case_pedantic = False - else: - # The user has expressed a case sensitivity choice, but we don't - # know the case sensitivity of the underlying filesystem, so we - # must use scandir() for everything, including non-wildcard parts. - case_pedantic = True - recursive = True if recurse_symlinks else _no_recurse_symlinks - globber = self._globber(self.parser.sep, case_sensitive, case_pedantic, recursive) - return globber.selector(parts) - - def glob(self, pattern, *, case_sensitive=None, recurse_symlinks=True): - """Iterate over this subtree and yield all existing files (of any - kind, including directories) matching the given relative pattern. - """ - if not isinstance(pattern, PurePathBase): - pattern = self.with_segments(pattern) - anchor, parts = pattern._stack - if anchor: - raise NotImplementedError("Non-relative patterns are unsupported") - select = self._glob_selector(parts, case_sensitive, recurse_symlinks) - return select(self) - - def rglob(self, pattern, *, case_sensitive=None, recurse_symlinks=True): - """Recursively yield all existing files (of any kind, including - directories) matching the given relative pattern, anywhere in - this subtree. - """ - if not isinstance(pattern, PurePathBase): - pattern = self.with_segments(pattern) - pattern = '**' / pattern - return self.glob(pattern, case_sensitive=case_sensitive, recurse_symlinks=recurse_symlinks) - - def walk(self, top_down=True, on_error=None, follow_symlinks=False): - """Walk the directory tree from this directory, similar to os.walk().""" - paths = [self] - while paths: - path = paths.pop() - if isinstance(path, tuple): - yield path - continue - dirnames = [] - filenames = [] - if not top_down: - paths.append((path, dirnames, filenames)) - try: - for child in path.iterdir(): - try: - if child.is_dir(follow_symlinks=follow_symlinks): - if not top_down: - paths.append(child) - dirnames.append(child.name) - else: - filenames.append(child.name) - except OSError: - filenames.append(child.name) - except OSError as error: - if on_error is not None: - on_error(error) - if not top_down: - while not isinstance(paths.pop(), tuple): - pass - continue - if top_down: - yield path, dirnames, filenames - paths += [path.joinpath(d) for d in reversed(dirnames)] - - def absolute(self): - """Return an absolute version of this path - No normalization or symlink resolution is performed. - - Use resolve() to resolve symlinks and remove '..' segments. - """ - raise UnsupportedOperation(self._unsupported_msg('absolute()')) - - @classmethod - def cwd(cls): - """Return a new path pointing to the current working directory.""" - # We call 'absolute()' rather than using 'os.getcwd()' directly to - # enable users to replace the implementation of 'absolute()' in a - # subclass and benefit from the new behaviour here. This works because - # os.path.abspath('.') == os.getcwd(). - return cls('').absolute() - - def expanduser(self): - """ Return a new path with expanded ~ and ~user constructs - (as returned by os.path.expanduser) - """ - raise UnsupportedOperation(self._unsupported_msg('expanduser()')) - - @classmethod - def home(cls): - """Return a new path pointing to expanduser('~'). - """ - return cls("~").expanduser() - - def readlink(self): - """ - Return the path to which the symbolic link points. - """ - raise UnsupportedOperation(self._unsupported_msg('readlink()')) - readlink._supported = False - - def resolve(self, strict=False): - """ - Make the path absolute, resolving all symlinks on the way and also - normalizing it. - """ - if self._resolving: - return self - path_root, parts = self._stack - path = self.with_segments(path_root) - try: - path = path.absolute() - except UnsupportedOperation: - path_tail = [] - else: - path_root, path_tail = path._stack - path_tail.reverse() - - # If the user has *not* overridden the `readlink()` method, then symlinks are unsupported - # and (in non-strict mode) we can improve performance by not calling `stat()`. - querying = strict or getattr(self.readlink, '_supported', True) - link_count = 0 - while parts: - part = parts.pop() - if not part or part == '.': - continue - if part == '..': - if not path_tail: - if path_root: - # Delete '..' segment immediately following root - continue - elif path_tail[-1] != '..': - # Delete '..' segment and its predecessor - path_tail.pop() - continue - path_tail.append(part) - if querying and part != '..': - path = self.with_segments(path_root + self.parser.sep.join(path_tail)) - path._resolving = True - try: - st = path.stat(follow_symlinks=False) - if S_ISLNK(st.st_mode): - # Like Linux and macOS, raise OSError(errno.ELOOP) if too many symlinks are - # encountered during resolution. - link_count += 1 - if link_count >= self._max_symlinks: - raise OSError(ELOOP, "Too many symbolic links in path", self._raw_path) - target_root, target_parts = path.readlink()._stack - # If the symlink target is absolute (like '/etc/hosts'), set the current - # path to its uppermost parent (like '/'). - if target_root: - path_root = target_root - path_tail.clear() - else: - path_tail.pop() - # Add the symlink target's reversed tail parts (like ['hosts', 'etc']) to - # the stack of unresolved path parts. - parts.extend(target_parts) - continue - elif parts and not S_ISDIR(st.st_mode): - raise NotADirectoryError(ENOTDIR, "Not a directory", self._raw_path) - except OSError: - if strict: - raise - else: - querying = False - return self.with_segments(path_root + self.parser.sep.join(path_tail)) - - def symlink_to(self, target, target_is_directory=False): - """ - Make this path a symlink pointing to the target path. - Note the order of arguments (link, target) is the reverse of os.symlink. - """ - raise UnsupportedOperation(self._unsupported_msg('symlink_to()')) - - def hardlink_to(self, target): - """ - Make this path a hard link pointing to the same file as *target*. - - Note the order of arguments (self, target) is the reverse of os.link's. - """ - raise UnsupportedOperation(self._unsupported_msg('hardlink_to()')) - - def touch(self, mode=0o666, exist_ok=True): - """ - Create this file with the given access mode, if it doesn't exist. - """ - raise UnsupportedOperation(self._unsupported_msg('touch()')) - - def mkdir(self, mode=0o777, parents=False, exist_ok=False): - """ - Create a new directory at this given path. - """ - raise UnsupportedOperation(self._unsupported_msg('mkdir()')) - - def rename(self, target): - """ - Rename this path to the target path. - - The target path may be absolute or relative. Relative paths are - interpreted relative to the current working directory, *not* the - directory of the Path object. - - Returns the new Path instance pointing to the target path. - """ - raise UnsupportedOperation(self._unsupported_msg('rename()')) - - def replace(self, target): - """ - Rename this path to the target path, overwriting if that path exists. - - The target path may be absolute or relative. Relative paths are - interpreted relative to the current working directory, *not* the - directory of the Path object. - - Returns the new Path instance pointing to the target path. - """ - raise UnsupportedOperation(self._unsupported_msg('replace()')) - - def chmod(self, mode, *, follow_symlinks=True): - """ - Change the permissions of the path, like os.chmod(). - """ - raise UnsupportedOperation(self._unsupported_msg('chmod()')) - - def lchmod(self, mode): - """ - Like chmod(), except if the path points to a symlink, the symlink's - permissions are changed, rather than its target's. - """ - self.chmod(mode, follow_symlinks=False) - - def unlink(self, missing_ok=False): - """ - Remove this file or link. - If the path is a directory, use rmdir() instead. - """ - raise UnsupportedOperation(self._unsupported_msg('unlink()')) - - def rmdir(self): - """ - Remove this directory. The directory must be empty. - """ - raise UnsupportedOperation(self._unsupported_msg('rmdir()')) - - def owner(self, *, follow_symlinks=True): - """ - Return the login name of the file owner. - """ - raise UnsupportedOperation(self._unsupported_msg('owner()')) - - def group(self, *, follow_symlinks=True): - """ - Return the group name of the file gid. - """ - raise UnsupportedOperation(self._unsupported_msg('group()')) - - @classmethod - def from_uri(cls, uri): - """Return a new path from the given 'file' URI.""" - raise UnsupportedOperation(cls._unsupported_msg('from_uri()')) - - def as_uri(self): - """Return the path as a URI.""" - raise UnsupportedOperation(self._unsupported_msg('as_uri()')) diff --git a/Lib/pathlib/_local.py b/Lib/pathlib/_local.py index 0188e7c7722..58e137f2a92 100644 --- a/Lib/pathlib/_local.py +++ b/Lib/pathlib/_local.py @@ -1,861 +1,12 @@ -import io -import ntpath -import operator -import os -import posixpath -import sys -import warnings -from glob import _StringGlobber -from itertools import chain -from _collections_abc import Sequence - -try: - import pwd -except ImportError: - pwd = None -try: - import grp -except ImportError: - grp = None - -from ._abc import UnsupportedOperation, PurePathBase, PathBase +""" +This module exists so that pathlib objects pickled under Python 3.13 can be +unpickled in 3.14+. +""" +from pathlib import * __all__ = [ + "UnsupportedOperation", "PurePath", "PurePosixPath", "PureWindowsPath", "Path", "PosixPath", "WindowsPath", - ] - - -class _PathParents(Sequence): - """This object provides sequence-like access to the logical ancestors - of a path. Don't try to construct it yourself.""" - __slots__ = ('_path', '_drv', '_root', '_tail') - - def __init__(self, path): - self._path = path - self._drv = path.drive - self._root = path.root - self._tail = path._tail - - def __len__(self): - return len(self._tail) - - def __getitem__(self, idx): - if isinstance(idx, slice): - return tuple(self[i] for i in range(*idx.indices(len(self)))) - - if idx >= len(self) or idx < -len(self): - raise IndexError(idx) - if idx < 0: - idx += len(self) - return self._path._from_parsed_parts(self._drv, self._root, - self._tail[:-idx - 1]) - - def __repr__(self): - return "<{}.parents>".format(type(self._path).__name__) - - -class PurePath(PurePathBase): - """Base class for manipulating paths without I/O. - - PurePath represents a filesystem path and offers operations which - don't imply any actual filesystem I/O. Depending on your system, - instantiating a PurePath will return either a PurePosixPath or a - PureWindowsPath object. You can also instantiate either of these classes - directly, regardless of your system. - """ - - __slots__ = ( - # The `_raw_paths` slot stores unnormalized string paths. This is set - # in the `__init__()` method. - '_raw_paths', - - # The `_drv`, `_root` and `_tail_cached` slots store parsed and - # normalized parts of the path. They are set when any of the `drive`, - # `root` or `_tail` properties are accessed for the first time. The - # three-part division corresponds to the result of - # `os.path.splitroot()`, except that the tail is further split on path - # separators (i.e. it is a list of strings), and that the root and - # tail are normalized. - '_drv', '_root', '_tail_cached', - - # The `_str` slot stores the string representation of the path, - # computed from the drive, root and tail when `__str__()` is called - # for the first time. It's used to implement `_str_normcase` - '_str', - - # The `_str_normcase_cached` slot stores the string path with - # normalized case. It is set when the `_str_normcase` property is - # accessed for the first time. It's used to implement `__eq__()` - # `__hash__()`, and `_parts_normcase` - '_str_normcase_cached', - - # The `_parts_normcase_cached` slot stores the case-normalized - # string path after splitting on path separators. It's set when the - # `_parts_normcase` property is accessed for the first time. It's used - # to implement comparison methods like `__lt__()`. - '_parts_normcase_cached', - - # The `_hash` slot stores the hash of the case-normalized string - # path. It's set when `__hash__()` is called for the first time. - '_hash', - ) - parser = os.path - _globber = _StringGlobber - - def __new__(cls, *args, **kwargs): - """Construct a PurePath from one or several strings and or existing - PurePath objects. The strings and path objects are combined so as - to yield a canonicalized path, which is incorporated into the - new PurePath object. - """ - if cls is PurePath: - cls = PureWindowsPath if os.name == 'nt' else PurePosixPath - return object.__new__(cls) - - def __init__(self, *args): - paths = [] - for arg in args: - if isinstance(arg, PurePath): - if arg.parser is not self.parser: - # GH-103631: Convert separators for backwards compatibility. - paths.append(arg.as_posix()) - else: - paths.extend(arg._raw_paths) - else: - try: - path = os.fspath(arg) - except TypeError: - path = arg - if not isinstance(path, str): - raise TypeError( - "argument should be a str or an os.PathLike " - "object where __fspath__ returns a str, " - f"not {type(path).__name__!r}") - paths.append(path) - # Avoid calling super().__init__, as an optimisation - self._raw_paths = paths - - def joinpath(self, *pathsegments): - """Combine this path with one or several arguments, and return a - new path representing either a subpath (if all arguments are relative - paths) or a totally different path (if one of the arguments is - anchored). - """ - return self.with_segments(self, *pathsegments) - - def __truediv__(self, key): - try: - return self.with_segments(self, key) - except TypeError: - return NotImplemented - - def __rtruediv__(self, key): - try: - return self.with_segments(key, self) - except TypeError: - return NotImplemented - - def __reduce__(self): - return self.__class__, tuple(self._raw_paths) - - def __repr__(self): - return "{}({!r})".format(self.__class__.__name__, self.as_posix()) - - def __fspath__(self): - return str(self) - - def __bytes__(self): - """Return the bytes representation of the path. This is only - recommended to use under Unix.""" - return os.fsencode(self) - - @property - def _str_normcase(self): - # String with normalized case, for hashing and equality checks - try: - return self._str_normcase_cached - except AttributeError: - if self.parser is posixpath: - self._str_normcase_cached = str(self) - else: - self._str_normcase_cached = str(self).lower() - return self._str_normcase_cached - - def __hash__(self): - try: - return self._hash - except AttributeError: - self._hash = hash(self._str_normcase) - return self._hash - - def __eq__(self, other): - if not isinstance(other, PurePath): - return NotImplemented - return self._str_normcase == other._str_normcase and self.parser is other.parser - - @property - def _parts_normcase(self): - # Cached parts with normalized case, for comparisons. - try: - return self._parts_normcase_cached - except AttributeError: - self._parts_normcase_cached = self._str_normcase.split(self.parser.sep) - return self._parts_normcase_cached - - def __lt__(self, other): - if not isinstance(other, PurePath) or self.parser is not other.parser: - return NotImplemented - return self._parts_normcase < other._parts_normcase - - def __le__(self, other): - if not isinstance(other, PurePath) or self.parser is not other.parser: - return NotImplemented - return self._parts_normcase <= other._parts_normcase - - def __gt__(self, other): - if not isinstance(other, PurePath) or self.parser is not other.parser: - return NotImplemented - return self._parts_normcase > other._parts_normcase - - def __ge__(self, other): - if not isinstance(other, PurePath) or self.parser is not other.parser: - return NotImplemented - return self._parts_normcase >= other._parts_normcase - - def __str__(self): - """Return the string representation of the path, suitable for - passing to system calls.""" - try: - return self._str - except AttributeError: - self._str = self._format_parsed_parts(self.drive, self.root, - self._tail) or '.' - return self._str - - @classmethod - def _format_parsed_parts(cls, drv, root, tail): - if drv or root: - return drv + root + cls.parser.sep.join(tail) - elif tail and cls.parser.splitdrive(tail[0])[0]: - tail = ['.'] + tail - return cls.parser.sep.join(tail) - - def _from_parsed_parts(self, drv, root, tail): - path = self._from_parsed_string(self._format_parsed_parts(drv, root, tail)) - path._drv = drv - path._root = root - path._tail_cached = tail - return path - - def _from_parsed_string(self, path_str): - path = self.with_segments(path_str) - path._str = path_str or '.' - return path - - @classmethod - def _parse_path(cls, path): - if not path: - return '', '', [] - sep = cls.parser.sep - altsep = cls.parser.altsep - if altsep: - path = path.replace(altsep, sep) - drv, root, rel = cls.parser.splitroot(path) - if not root and drv.startswith(sep) and not drv.endswith(sep): - drv_parts = drv.split(sep) - if len(drv_parts) == 4 and drv_parts[2] not in '?.': - # e.g. //server/share - root = sep - elif len(drv_parts) == 6: - # e.g. //?/unc/server/share - root = sep - parsed = [sys.intern(str(x)) for x in rel.split(sep) if x and x != '.'] - return drv, root, parsed - - @property - def _raw_path(self): - """The joined but unnormalized path.""" - paths = self._raw_paths - if len(paths) == 0: - path = '' - elif len(paths) == 1: - path = paths[0] - else: - path = self.parser.join(*paths) - return path - - @property - def drive(self): - """The drive prefix (letter or UNC path), if any.""" - try: - return self._drv - except AttributeError: - self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) - return self._drv - - @property - def root(self): - """The root of the path, if any.""" - try: - return self._root - except AttributeError: - self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) - return self._root - - @property - def _tail(self): - try: - return self._tail_cached - except AttributeError: - self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) - return self._tail_cached - - @property - def anchor(self): - """The concatenation of the drive and root, or ''.""" - return self.drive + self.root - - @property - def parts(self): - """An object providing sequence-like access to the - components in the filesystem path.""" - if self.drive or self.root: - return (self.drive + self.root,) + tuple(self._tail) - else: - return tuple(self._tail) - - @property - def parent(self): - """The logical parent of the path.""" - drv = self.drive - root = self.root - tail = self._tail - if not tail: - return self - return self._from_parsed_parts(drv, root, tail[:-1]) - - @property - def parents(self): - """A sequence of this path's logical parents.""" - # The value of this property should not be cached on the path object, - # as doing so would introduce a reference cycle. - return _PathParents(self) - - @property - def name(self): - """The final path component, if any.""" - tail = self._tail - if not tail: - return '' - return tail[-1] - - def with_name(self, name): - """Return a new path with the file name changed.""" - p = self.parser - if not name or p.sep in name or (p.altsep and p.altsep in name) or name == '.': - raise ValueError(f"Invalid name {name!r}") - tail = self._tail.copy() - if not tail: - raise ValueError(f"{self!r} has an empty name") - tail[-1] = name - return self._from_parsed_parts(self.drive, self.root, tail) - - def relative_to(self, other, /, *_deprecated, walk_up=False): - """Return the relative path to another path identified by the passed - arguments. If the operation is not possible (because this is not - related to the other path), raise ValueError. - - The *walk_up* parameter controls whether `..` may be used to resolve - the path. - """ - if _deprecated: - msg = ("support for supplying more than one positional argument " - "to pathlib.PurePath.relative_to() is deprecated and " - "scheduled for removal in Python 3.14") - warnings.warn(msg, DeprecationWarning, stacklevel=2) - other = self.with_segments(other, *_deprecated) - elif not isinstance(other, PurePath): - other = self.with_segments(other) - for step, path in enumerate(chain([other], other.parents)): - if path == self or path in self.parents: - break - elif not walk_up: - raise ValueError(f"{str(self)!r} is not in the subpath of {str(other)!r}") - elif path.name == '..': - raise ValueError(f"'..' segment in {str(other)!r} cannot be walked") - else: - raise ValueError(f"{str(self)!r} and {str(other)!r} have different anchors") - parts = ['..'] * step + self._tail[len(path._tail):] - return self._from_parsed_parts('', '', parts) - - def is_relative_to(self, other, /, *_deprecated): - """Return True if the path is relative to another path or False. - """ - if _deprecated: - msg = ("support for supplying more than one argument to " - "pathlib.PurePath.is_relative_to() is deprecated and " - "scheduled for removal in Python 3.14") - warnings.warn(msg, DeprecationWarning, stacklevel=2) - other = self.with_segments(other, *_deprecated) - elif not isinstance(other, PurePath): - other = self.with_segments(other) - return other == self or other in self.parents - - def is_absolute(self): - """True if the path is absolute (has both a root and, if applicable, - a drive).""" - if self.parser is posixpath: - # Optimization: work with raw paths on POSIX. - for path in self._raw_paths: - if path.startswith('/'): - return True - return False - return self.parser.isabs(self) - - def is_reserved(self): - """Return True if the path contains one of the special names reserved - by the system, if any.""" - msg = ("pathlib.PurePath.is_reserved() is deprecated and scheduled " - "for removal in Python 3.15. Use os.path.isreserved() to " - "detect reserved paths on Windows.") - warnings.warn(msg, DeprecationWarning, stacklevel=2) - if self.parser is ntpath: - return self.parser.isreserved(self) - return False - - def as_uri(self): - """Return the path as a URI.""" - if not self.is_absolute(): - raise ValueError("relative path can't be expressed as a file URI") - - drive = self.drive - if len(drive) == 2 and drive[1] == ':': - # It's a path on a local drive => 'file:///c:/a/b' - prefix = 'file:///' + drive - path = self.as_posix()[2:] - elif drive: - # It's a path on a network drive => 'file://host/share/a/b' - prefix = 'file:' - path = self.as_posix() - else: - # It's a posix path => 'file:///etc/hosts' - prefix = 'file://' - path = str(self) - from urllib.parse import quote_from_bytes - return prefix + quote_from_bytes(os.fsencode(path)) - - @property - def _pattern_str(self): - """The path expressed as a string, for use in pattern-matching.""" - # The string representation of an empty path is a single dot ('.'). Empty - # paths shouldn't match wildcards, so we change it to the empty string. - path_str = str(self) - return '' if path_str == '.' else path_str - -# Subclassing os.PathLike makes isinstance() checks slower, -# which in turn makes Path construction slower. Register instead! -os.PathLike.register(PurePath) - - -class PurePosixPath(PurePath): - """PurePath subclass for non-Windows systems. - - On a POSIX system, instantiating a PurePath should return this object. - However, you can also instantiate it directly on any system. - """ - parser = posixpath - __slots__ = () - - -class PureWindowsPath(PurePath): - """PurePath subclass for Windows systems. - - On a Windows system, instantiating a PurePath should return this object. - However, you can also instantiate it directly on any system. - """ - parser = ntpath - __slots__ = () - - -class Path(PathBase, PurePath): - """PurePath subclass that can make system calls. - - Path represents a filesystem path but unlike PurePath, also offers - methods to do system calls on path objects. Depending on your system, - instantiating a Path will return either a PosixPath or a WindowsPath - object. You can also instantiate a PosixPath or WindowsPath directly, - but cannot instantiate a WindowsPath on a POSIX system or vice versa. - """ - __slots__ = () - as_uri = PurePath.as_uri - - @classmethod - def _unsupported_msg(cls, attribute): - return f"{cls.__name__}.{attribute} is unsupported on this system" - - def __init__(self, *args, **kwargs): - if kwargs: - msg = ("support for supplying keyword arguments to pathlib.PurePath " - "is deprecated and scheduled for removal in Python {remove}") - warnings._deprecated("pathlib.PurePath(**kwargs)", msg, remove=(3, 14)) - super().__init__(*args) - - def __new__(cls, *args, **kwargs): - if cls is Path: - cls = WindowsPath if os.name == 'nt' else PosixPath - return object.__new__(cls) - - def stat(self, *, follow_symlinks=True): - """ - Return the result of the stat() system call on this path, like - os.stat() does. - """ - return os.stat(self, follow_symlinks=follow_symlinks) - - def is_mount(self): - """ - Check if this path is a mount point - """ - return os.path.ismount(self) - - def is_junction(self): - """ - Whether this path is a junction. - """ - return os.path.isjunction(self) - - def open(self, mode='r', buffering=-1, encoding=None, - errors=None, newline=None): - """ - Open the file pointed to by this path and return a file object, as - the built-in open() function does. - """ - if "b" not in mode: - encoding = io.text_encoding(encoding) - return io.open(self, mode, buffering, encoding, errors, newline) - - def read_text(self, encoding=None, errors=None, newline=None): - """ - Open the file in text mode, read it, and close the file. - """ - # Call io.text_encoding() here to ensure any warning is raised at an - # appropriate stack level. - encoding = io.text_encoding(encoding) - return PathBase.read_text(self, encoding, errors, newline) - - def write_text(self, data, encoding=None, errors=None, newline=None): - """ - Open the file in text mode, write to it, and close the file. - """ - # Call io.text_encoding() here to ensure any warning is raised at an - # appropriate stack level. - encoding = io.text_encoding(encoding) - return PathBase.write_text(self, data, encoding, errors, newline) - - _remove_leading_dot = operator.itemgetter(slice(2, None)) - _remove_trailing_slash = operator.itemgetter(slice(-1)) - - def _filter_trailing_slash(self, paths): - sep = self.parser.sep - anchor_len = len(self.anchor) - for path_str in paths: - if len(path_str) > anchor_len and path_str[-1] == sep: - path_str = path_str[:-1] - yield path_str - - def iterdir(self): - """Yield path objects of the directory contents. - - The children are yielded in arbitrary order, and the - special entries '.' and '..' are not included. - """ - root_dir = str(self) - with os.scandir(root_dir) as scandir_it: - paths = [entry.path for entry in scandir_it] - if root_dir == '.': - paths = map(self._remove_leading_dot, paths) - return map(self._from_parsed_string, paths) - - def glob(self, pattern, *, case_sensitive=None, recurse_symlinks=False): - """Iterate over this subtree and yield all existing files (of any - kind, including directories) matching the given relative pattern. - """ - sys.audit("pathlib.Path.glob", self, pattern) - if not isinstance(pattern, PurePath): - pattern = self.with_segments(pattern) - if pattern.anchor: - raise NotImplementedError("Non-relative patterns are unsupported") - parts = pattern._tail.copy() - if not parts: - raise ValueError("Unacceptable pattern: {!r}".format(pattern)) - raw = pattern._raw_path - if raw[-1] in (self.parser.sep, self.parser.altsep): - # GH-65238: pathlib doesn't preserve trailing slash. Add it back. - parts.append('') - select = self._glob_selector(parts[::-1], case_sensitive, recurse_symlinks) - root = str(self) - paths = select(root) - - # Normalize results - if root == '.': - paths = map(self._remove_leading_dot, paths) - if parts[-1] == '': - paths = map(self._remove_trailing_slash, paths) - elif parts[-1] == '**': - paths = self._filter_trailing_slash(paths) - paths = map(self._from_parsed_string, paths) - return paths - - def rglob(self, pattern, *, case_sensitive=None, recurse_symlinks=False): - """Recursively yield all existing files (of any kind, including - directories) matching the given relative pattern, anywhere in - this subtree. - """ - sys.audit("pathlib.Path.rglob", self, pattern) - if not isinstance(pattern, PurePath): - pattern = self.with_segments(pattern) - pattern = '**' / pattern - return self.glob(pattern, case_sensitive=case_sensitive, recurse_symlinks=recurse_symlinks) - - def walk(self, top_down=True, on_error=None, follow_symlinks=False): - """Walk the directory tree from this directory, similar to os.walk().""" - sys.audit("pathlib.Path.walk", self, on_error, follow_symlinks) - root_dir = str(self) - if not follow_symlinks: - follow_symlinks = os._walk_symlinks_as_files - results = os.walk(root_dir, top_down, on_error, follow_symlinks) - for path_str, dirnames, filenames in results: - if root_dir == '.': - path_str = path_str[2:] - yield self._from_parsed_string(path_str), dirnames, filenames - - def absolute(self): - """Return an absolute version of this path - No normalization or symlink resolution is performed. - - Use resolve() to resolve symlinks and remove '..' segments. - """ - if self.is_absolute(): - return self - if self.root: - drive = os.path.splitroot(os.getcwd())[0] - return self._from_parsed_parts(drive, self.root, self._tail) - if self.drive: - # There is a CWD on each drive-letter drive. - cwd = os.path.abspath(self.drive) - else: - cwd = os.getcwd() - if not self._tail: - # Fast path for "empty" paths, e.g. Path("."), Path("") or Path(). - # We pass only one argument to with_segments() to avoid the cost - # of joining, and we exploit the fact that getcwd() returns a - # fully-normalized string by storing it in _str. This is used to - # implement Path.cwd(). - return self._from_parsed_string(cwd) - drive, root, rel = os.path.splitroot(cwd) - if not rel: - return self._from_parsed_parts(drive, root, self._tail) - tail = rel.split(self.parser.sep) - tail.extend(self._tail) - return self._from_parsed_parts(drive, root, tail) - - def resolve(self, strict=False): - """ - Make the path absolute, resolving all symlinks on the way and also - normalizing it. - """ - - return self.with_segments(os.path.realpath(self, strict=strict)) - - if pwd: - def owner(self, *, follow_symlinks=True): - """ - Return the login name of the file owner. - """ - uid = self.stat(follow_symlinks=follow_symlinks).st_uid - return pwd.getpwuid(uid).pw_name - - if grp: - def group(self, *, follow_symlinks=True): - """ - Return the group name of the file gid. - """ - gid = self.stat(follow_symlinks=follow_symlinks).st_gid - return grp.getgrgid(gid).gr_name - - if hasattr(os, "readlink"): - def readlink(self): - """ - Return the path to which the symbolic link points. - """ - return self.with_segments(os.readlink(self)) - - def touch(self, mode=0o666, exist_ok=True): - """ - Create this file with the given access mode, if it doesn't exist. - """ - - if exist_ok: - # First try to bump modification time - # Implementation note: GNU touch uses the UTIME_NOW option of - # the utimensat() / futimens() functions. - try: - os.utime(self, None) - except OSError: - # Avoid exception chaining - pass - else: - return - flags = os.O_CREAT | os.O_WRONLY - if not exist_ok: - flags |= os.O_EXCL - fd = os.open(self, flags, mode) - os.close(fd) - - def mkdir(self, mode=0o777, parents=False, exist_ok=False): - """ - Create a new directory at this given path. - """ - try: - os.mkdir(self, mode) - except FileNotFoundError: - if not parents or self.parent == self: - raise - self.parent.mkdir(parents=True, exist_ok=True) - self.mkdir(mode, parents=False, exist_ok=exist_ok) - except OSError: - # Cannot rely on checking for EEXIST, since the operating system - # could give priority to other errors like EACCES or EROFS - if not exist_ok or not self.is_dir(): - raise - - def chmod(self, mode, *, follow_symlinks=True): - """ - Change the permissions of the path, like os.chmod(). - """ - os.chmod(self, mode, follow_symlinks=follow_symlinks) - - def unlink(self, missing_ok=False): - """ - Remove this file or link. - If the path is a directory, use rmdir() instead. - """ - try: - os.unlink(self) - except FileNotFoundError: - if not missing_ok: - raise - - def rmdir(self): - """ - Remove this directory. The directory must be empty. - """ - os.rmdir(self) - - def rename(self, target): - """ - Rename this path to the target path. - - The target path may be absolute or relative. Relative paths are - interpreted relative to the current working directory, *not* the - directory of the Path object. - - Returns the new Path instance pointing to the target path. - """ - os.rename(self, target) - return self.with_segments(target) - - def replace(self, target): - """ - Rename this path to the target path, overwriting if that path exists. - - The target path may be absolute or relative. Relative paths are - interpreted relative to the current working directory, *not* the - directory of the Path object. - - Returns the new Path instance pointing to the target path. - """ - os.replace(self, target) - return self.with_segments(target) - - if hasattr(os, "symlink"): - def symlink_to(self, target, target_is_directory=False): - """ - Make this path a symlink pointing to the target path. - Note the order of arguments (link, target) is the reverse of os.symlink. - """ - os.symlink(target, self, target_is_directory) - - if hasattr(os, "link"): - def hardlink_to(self, target): - """ - Make this path a hard link pointing to the same file as *target*. - - Note the order of arguments (self, target) is the reverse of os.link's. - """ - os.link(target, self) - - def expanduser(self): - """ Return a new path with expanded ~ and ~user constructs - (as returned by os.path.expanduser) - """ - if (not (self.drive or self.root) and - self._tail and self._tail[0][:1] == '~'): - homedir = os.path.expanduser(self._tail[0]) - if homedir[:1] == "~": - raise RuntimeError("Could not determine home directory.") - drv, root, tail = self._parse_path(homedir) - return self._from_parsed_parts(drv, root, tail + self._tail[1:]) - - return self - - @classmethod - def from_uri(cls, uri): - """Return a new path from the given 'file' URI.""" - if not uri.startswith('file:'): - raise ValueError(f"URI does not start with 'file:': {uri!r}") - path = uri[5:] - if path[:3] == '///': - # Remove empty authority - path = path[2:] - elif path[:12] == '//localhost/': - # Remove 'localhost' authority - path = path[11:] - if path[:3] == '///' or (path[:1] == '/' and path[2:3] in ':|'): - # Remove slash before DOS device/UNC path - path = path[1:] - if path[1:2] == '|': - # Replace bar with colon in DOS drive - path = path[:1] + ':' + path[2:] - from urllib.parse import unquote_to_bytes - path = cls(os.fsdecode(unquote_to_bytes(path))) - if not path.is_absolute(): - raise ValueError(f"URI is not absolute: {uri!r}") - return path - - -class PosixPath(Path, PurePosixPath): - """Path subclass for non-Windows systems. - - On a POSIX system, instantiating a Path should return this object. - """ - __slots__ = () - - if os.name == 'nt': - def __new__(cls, *args, **kwargs): - raise UnsupportedOperation( - f"cannot instantiate {cls.__name__!r} on your system") - -class WindowsPath(Path, PureWindowsPath): - """Path subclass for Windows systems. - - On a Windows system, instantiating a Path should return this object. - """ - __slots__ = () - - if os.name != 'nt': - def __new__(cls, *args, **kwargs): - raise UnsupportedOperation( - f"cannot instantiate {cls.__name__!r} on your system") +] diff --git a/Lib/pathlib/_os.py b/Lib/pathlib/_os.py new file mode 100644 index 00000000000..039836941dd --- /dev/null +++ b/Lib/pathlib/_os.py @@ -0,0 +1,530 @@ +""" +Low-level OS functionality wrappers used by pathlib. +""" + +from errno import * +from io import TextIOWrapper, text_encoding +from stat import S_ISDIR, S_ISREG, S_ISLNK, S_IMODE +import os +import sys +try: + import fcntl +except ImportError: + fcntl = None +try: + import posix +except ImportError: + posix = None +try: + import _winapi +except ImportError: + _winapi = None + + +def _get_copy_blocksize(infd): + """Determine blocksize for fastcopying on Linux. + Hopefully the whole file will be copied in a single call. + The copying itself should be performed in a loop 'till EOF is + reached (0 return) so a blocksize smaller or bigger than the actual + file size should not make any difference, also in case the file + content changes while being copied. + """ + try: + blocksize = max(os.fstat(infd).st_size, 2 ** 23) # min 8 MiB + except OSError: + blocksize = 2 ** 27 # 128 MiB + # On 32-bit architectures truncate to 1 GiB to avoid OverflowError, + # see gh-82500. + if sys.maxsize < 2 ** 32: + blocksize = min(blocksize, 2 ** 30) + return blocksize + + +if fcntl and hasattr(fcntl, 'FICLONE'): + def _ficlone(source_fd, target_fd): + """ + Perform a lightweight copy of two files, where the data blocks are + copied only when modified. This is known as Copy on Write (CoW), + instantaneous copy or reflink. + """ + fcntl.ioctl(target_fd, fcntl.FICLONE, source_fd) +else: + _ficlone = None + + +if posix and hasattr(posix, '_fcopyfile'): + def _fcopyfile(source_fd, target_fd): + """ + Copy a regular file content using high-performance fcopyfile(3) + syscall (macOS). + """ + posix._fcopyfile(source_fd, target_fd, posix._COPYFILE_DATA) +else: + _fcopyfile = None + + +if hasattr(os, 'copy_file_range'): + def _copy_file_range(source_fd, target_fd): + """ + Copy data from one regular mmap-like fd to another by using a + high-performance copy_file_range(2) syscall that gives filesystems + an opportunity to implement the use of reflinks or server-side + copy. + This should work on Linux >= 4.5 only. + """ + blocksize = _get_copy_blocksize(source_fd) + offset = 0 + while True: + sent = os.copy_file_range(source_fd, target_fd, blocksize, + offset_dst=offset) + if sent == 0: + break # EOF + offset += sent +else: + _copy_file_range = None + + +if hasattr(os, 'sendfile'): + def _sendfile(source_fd, target_fd): + """Copy data from one regular mmap-like fd to another by using + high-performance sendfile(2) syscall. + This should work on Linux >= 2.6.33 only. + """ + blocksize = _get_copy_blocksize(source_fd) + offset = 0 + while True: + sent = os.sendfile(target_fd, source_fd, offset, blocksize) + if sent == 0: + break # EOF + offset += sent +else: + _sendfile = None + + +if _winapi and hasattr(_winapi, 'CopyFile2'): + def copyfile2(source, target): + """ + Copy from one file to another using CopyFile2 (Windows only). + """ + _winapi.CopyFile2(source, target, 0) +else: + copyfile2 = None + + +def copyfileobj(source_f, target_f): + """ + Copy data from file-like object source_f to file-like object target_f. + """ + try: + source_fd = source_f.fileno() + target_fd = target_f.fileno() + except Exception: + pass # Fall through to generic code. + else: + try: + # Use OS copy-on-write where available. + if _ficlone: + try: + _ficlone(source_fd, target_fd) + return + except OSError as err: + if err.errno not in (EBADF, EOPNOTSUPP, ETXTBSY, EXDEV): + raise err + + # Use OS copy where available. + if _fcopyfile: + try: + _fcopyfile(source_fd, target_fd) + return + except OSError as err: + if err.errno not in (EINVAL, ENOTSUP): + raise err + if _copy_file_range: + try: + _copy_file_range(source_fd, target_fd) + return + except OSError as err: + if err.errno not in (ETXTBSY, EXDEV): + raise err + if _sendfile: + try: + _sendfile(source_fd, target_fd) + return + except OSError as err: + if err.errno != ENOTSOCK: + raise err + except OSError as err: + # Produce more useful error messages. + err.filename = source_f.name + err.filename2 = target_f.name + raise err + + # Last resort: copy with fileobj read() and write(). + read_source = source_f.read + write_target = target_f.write + while buf := read_source(1024 * 1024): + write_target(buf) + + +def magic_open(path, mode='r', buffering=-1, encoding=None, errors=None, + newline=None): + """ + Open the file pointed to by this path and return a file object, as + the built-in open() function does. + """ + text = 'b' not in mode + if text: + # Call io.text_encoding() here to ensure any warning is raised at an + # appropriate stack level. + encoding = text_encoding(encoding) + try: + return open(path, mode, buffering, encoding, errors, newline) + except TypeError: + pass + cls = type(path) + mode = ''.join(sorted(c for c in mode if c not in 'bt')) + if text: + try: + attr = getattr(cls, f'__open_{mode}__') + except AttributeError: + pass + else: + return attr(path, buffering, encoding, errors, newline) + elif encoding is not None: + raise ValueError("binary mode doesn't take an encoding argument") + elif errors is not None: + raise ValueError("binary mode doesn't take an errors argument") + elif newline is not None: + raise ValueError("binary mode doesn't take a newline argument") + + try: + attr = getattr(cls, f'__open_{mode}b__') + except AttributeError: + pass + else: + stream = attr(path, buffering) + if text: + stream = TextIOWrapper(stream, encoding, errors, newline) + return stream + + raise TypeError(f"{cls.__name__} can't be opened with mode {mode!r}") + + +def ensure_distinct_paths(source, target): + """ + Raise OSError(EINVAL) if the other path is within this path. + """ + # Note: there is no straightforward, foolproof algorithm to determine + # if one directory is within another (a particularly perverse example + # would be a single network share mounted in one location via NFS, and + # in another location via CIFS), so we simply checks whether the + # other path is lexically equal to, or within, this path. + if source == target: + err = OSError(EINVAL, "Source and target are the same path") + elif source in target.parents: + err = OSError(EINVAL, "Source path is a parent of target path") + else: + return + err.filename = str(source) + err.filename2 = str(target) + raise err + + +def ensure_different_files(source, target): + """ + Raise OSError(EINVAL) if both paths refer to the same file. + """ + try: + source_file_id = source.info._file_id + target_file_id = target.info._file_id + except AttributeError: + if source != target: + return + else: + try: + if source_file_id() != target_file_id(): + return + except (OSError, ValueError): + return + err = OSError(EINVAL, "Source and target are the same file") + err.filename = str(source) + err.filename2 = str(target) + raise err + + +def copy_info(info, target, follow_symlinks=True): + """Copy metadata from the given PathInfo to the given local path.""" + copy_times_ns = ( + hasattr(info, '_access_time_ns') and + hasattr(info, '_mod_time_ns') and + (follow_symlinks or os.utime in os.supports_follow_symlinks)) + if copy_times_ns: + t0 = info._access_time_ns(follow_symlinks=follow_symlinks) + t1 = info._mod_time_ns(follow_symlinks=follow_symlinks) + os.utime(target, ns=(t0, t1), follow_symlinks=follow_symlinks) + + # We must copy extended attributes before the file is (potentially) + # chmod()'ed read-only, otherwise setxattr() will error with -EACCES. + copy_xattrs = ( + hasattr(info, '_xattrs') and + hasattr(os, 'setxattr') and + (follow_symlinks or os.setxattr in os.supports_follow_symlinks)) + if copy_xattrs: + xattrs = info._xattrs(follow_symlinks=follow_symlinks) + for attr, value in xattrs: + try: + os.setxattr(target, attr, value, follow_symlinks=follow_symlinks) + except OSError as e: + if e.errno not in (EPERM, ENOTSUP, ENODATA, EINVAL, EACCES): + raise + + copy_posix_permissions = ( + hasattr(info, '_posix_permissions') and + (follow_symlinks or os.chmod in os.supports_follow_symlinks)) + if copy_posix_permissions: + posix_permissions = info._posix_permissions(follow_symlinks=follow_symlinks) + try: + os.chmod(target, posix_permissions, follow_symlinks=follow_symlinks) + except NotImplementedError: + # if we got a NotImplementedError, it's because + # * follow_symlinks=False, + # * lchown() is unavailable, and + # * either + # * fchownat() is unavailable or + # * fchownat() doesn't implement AT_SYMLINK_NOFOLLOW. + # (it returned ENOSUP.) + # therefore we're out of options--we simply cannot chown the + # symlink. give up, suppress the error. + # (which is what shutil always did in this circumstance.) + pass + + copy_bsd_flags = ( + hasattr(info, '_bsd_flags') and + hasattr(os, 'chflags') and + (follow_symlinks or os.chflags in os.supports_follow_symlinks)) + if copy_bsd_flags: + bsd_flags = info._bsd_flags(follow_symlinks=follow_symlinks) + try: + os.chflags(target, bsd_flags, follow_symlinks=follow_symlinks) + except OSError as why: + if why.errno not in (EOPNOTSUPP, ENOTSUP): + raise + + +class _PathInfoBase: + __slots__ = ('_path', '_stat_result', '_lstat_result') + + def __init__(self, path): + self._path = str(path) + + def __repr__(self): + path_type = "WindowsPath" if os.name == "nt" else "PosixPath" + return f"<{path_type}.info>" + + def _stat(self, *, follow_symlinks=True, ignore_errors=False): + """Return the status as an os.stat_result, or None if stat() fails and + ignore_errors is true.""" + if follow_symlinks: + try: + result = self._stat_result + except AttributeError: + pass + else: + if ignore_errors or result is not None: + return result + try: + self._stat_result = os.stat(self._path) + except (OSError, ValueError): + self._stat_result = None + if not ignore_errors: + raise + return self._stat_result + else: + try: + result = self._lstat_result + except AttributeError: + pass + else: + if ignore_errors or result is not None: + return result + try: + self._lstat_result = os.lstat(self._path) + except (OSError, ValueError): + self._lstat_result = None + if not ignore_errors: + raise + return self._lstat_result + + def _posix_permissions(self, *, follow_symlinks=True): + """Return the POSIX file permissions.""" + return S_IMODE(self._stat(follow_symlinks=follow_symlinks).st_mode) + + def _file_id(self, *, follow_symlinks=True): + """Returns the identifier of the file.""" + st = self._stat(follow_symlinks=follow_symlinks) + return st.st_dev, st.st_ino + + def _access_time_ns(self, *, follow_symlinks=True): + """Return the access time in nanoseconds.""" + return self._stat(follow_symlinks=follow_symlinks).st_atime_ns + + def _mod_time_ns(self, *, follow_symlinks=True): + """Return the modify time in nanoseconds.""" + return self._stat(follow_symlinks=follow_symlinks).st_mtime_ns + + if hasattr(os.stat_result, 'st_flags'): + def _bsd_flags(self, *, follow_symlinks=True): + """Return the flags.""" + return self._stat(follow_symlinks=follow_symlinks).st_flags + + if hasattr(os, 'listxattr'): + def _xattrs(self, *, follow_symlinks=True): + """Return the xattrs as a list of (attr, value) pairs, or an empty + list if extended attributes aren't supported.""" + try: + return [ + (attr, os.getxattr(self._path, attr, follow_symlinks=follow_symlinks)) + for attr in os.listxattr(self._path, follow_symlinks=follow_symlinks)] + except OSError as err: + if err.errno not in (EPERM, ENOTSUP, ENODATA, EINVAL, EACCES): + raise + return [] + + +class _WindowsPathInfo(_PathInfoBase): + """Implementation of pathlib.types.PathInfo that provides status + information for Windows paths. Don't try to construct it yourself.""" + __slots__ = ('_exists', '_is_dir', '_is_file', '_is_symlink') + + def exists(self, *, follow_symlinks=True): + """Whether this path exists.""" + if not follow_symlinks and self.is_symlink(): + return True + try: + return self._exists + except AttributeError: + if os.path.exists(self._path): + self._exists = True + return True + else: + self._exists = self._is_dir = self._is_file = False + return False + + def is_dir(self, *, follow_symlinks=True): + """Whether this path is a directory.""" + if not follow_symlinks and self.is_symlink(): + return False + try: + return self._is_dir + except AttributeError: + if os.path.isdir(self._path): + self._is_dir = self._exists = True + return True + else: + self._is_dir = False + return False + + def is_file(self, *, follow_symlinks=True): + """Whether this path is a regular file.""" + if not follow_symlinks and self.is_symlink(): + return False + try: + return self._is_file + except AttributeError: + if os.path.isfile(self._path): + self._is_file = self._exists = True + return True + else: + self._is_file = False + return False + + def is_symlink(self): + """Whether this path is a symbolic link.""" + try: + return self._is_symlink + except AttributeError: + self._is_symlink = os.path.islink(self._path) + return self._is_symlink + + +class _PosixPathInfo(_PathInfoBase): + """Implementation of pathlib.types.PathInfo that provides status + information for POSIX paths. Don't try to construct it yourself.""" + __slots__ = () + + def exists(self, *, follow_symlinks=True): + """Whether this path exists.""" + st = self._stat(follow_symlinks=follow_symlinks, ignore_errors=True) + if st is None: + return False + return True + + def is_dir(self, *, follow_symlinks=True): + """Whether this path is a directory.""" + st = self._stat(follow_symlinks=follow_symlinks, ignore_errors=True) + if st is None: + return False + return S_ISDIR(st.st_mode) + + def is_file(self, *, follow_symlinks=True): + """Whether this path is a regular file.""" + st = self._stat(follow_symlinks=follow_symlinks, ignore_errors=True) + if st is None: + return False + return S_ISREG(st.st_mode) + + def is_symlink(self): + """Whether this path is a symbolic link.""" + st = self._stat(follow_symlinks=False, ignore_errors=True) + if st is None: + return False + return S_ISLNK(st.st_mode) + + +PathInfo = _WindowsPathInfo if os.name == 'nt' else _PosixPathInfo + + +class DirEntryInfo(_PathInfoBase): + """Implementation of pathlib.types.PathInfo that provides status + information by querying a wrapped os.DirEntry object. Don't try to + construct it yourself.""" + __slots__ = ('_entry',) + + def __init__(self, entry): + super().__init__(entry.path) + self._entry = entry + + def _stat(self, *, follow_symlinks=True, ignore_errors=False): + try: + return self._entry.stat(follow_symlinks=follow_symlinks) + except OSError: + if not ignore_errors: + raise + return None + + def exists(self, *, follow_symlinks=True): + """Whether this path exists.""" + if not follow_symlinks: + return True + return self._stat(ignore_errors=True) is not None + + def is_dir(self, *, follow_symlinks=True): + """Whether this path is a directory.""" + try: + return self._entry.is_dir(follow_symlinks=follow_symlinks) + except OSError: + return False + + def is_file(self, *, follow_symlinks=True): + """Whether this path is a regular file.""" + try: + return self._entry.is_file(follow_symlinks=follow_symlinks) + except OSError: + return False + + def is_symlink(self): + """Whether this path is a symbolic link.""" + try: + return self._entry.is_symlink() + except OSError: + return False diff --git a/Lib/pathlib/types.py b/Lib/pathlib/types.py new file mode 100644 index 00000000000..d8f5c34a1a7 --- /dev/null +++ b/Lib/pathlib/types.py @@ -0,0 +1,430 @@ +""" +Protocols for supporting classes in pathlib. +""" + +# This module also provides abstract base classes for rich path objects. +# These ABCs are a *private* part of the Python standard library, but they're +# made available as a PyPI package called "pathlib-abc". It's possible they'll +# become an official part of the standard library in future. +# +# Three ABCs are provided -- _JoinablePath, _ReadablePath and _WritablePath + + +from abc import ABC, abstractmethod +from glob import _PathGlobber +from io import text_encoding +from pathlib._os import magic_open, ensure_distinct_paths, ensure_different_files, copyfileobj +from pathlib import PurePath, Path +from typing import Optional, Protocol, runtime_checkable + + +def _explode_path(path, split): + """ + Split the path into a 2-tuple (anchor, parts), where *anchor* is the + uppermost parent of the path (equivalent to path.parents[-1]), and + *parts* is a reversed list of parts following the anchor. + """ + parent, name = split(path) + names = [] + while path != parent: + names.append(name) + path = parent + parent, name = split(path) + return path, names + + +@runtime_checkable +class _PathParser(Protocol): + """Protocol for path parsers, which do low-level path manipulation. + + Path parsers provide a subset of the os.path API, specifically those + functions needed to provide JoinablePath functionality. Each JoinablePath + subclass references its path parser via a 'parser' class attribute. + """ + + sep: str + altsep: Optional[str] + def split(self, path: str) -> tuple[str, str]: ... + def splitext(self, path: str) -> tuple[str, str]: ... + def normcase(self, path: str) -> str: ... + + +@runtime_checkable +class PathInfo(Protocol): + """Protocol for path info objects, which support querying the file type. + Methods may return cached results. + """ + def exists(self, *, follow_symlinks: bool = True) -> bool: ... + def is_dir(self, *, follow_symlinks: bool = True) -> bool: ... + def is_file(self, *, follow_symlinks: bool = True) -> bool: ... + def is_symlink(self) -> bool: ... + + +class _JoinablePath(ABC): + """Abstract base class for pure path objects. + + This class *does not* provide several magic methods that are defined in + its implementation PurePath. They are: __init__, __fspath__, __bytes__, + __reduce__, __hash__, __eq__, __lt__, __le__, __gt__, __ge__. + """ + __slots__ = () + + @property + @abstractmethod + def parser(self): + """Implementation of pathlib._types.Parser used for low-level path + parsing and manipulation. + """ + raise NotImplementedError + + @abstractmethod + def with_segments(self, *pathsegments): + """Construct a new path object from any number of path-like objects. + Subclasses may override this method to customize how new path objects + are created from methods like `iterdir()`. + """ + raise NotImplementedError + + @abstractmethod + def __str__(self): + """Return the string representation of the path, suitable for + passing to system calls.""" + raise NotImplementedError + + @property + def anchor(self): + """The concatenation of the drive and root, or ''.""" + return _explode_path(str(self), self.parser.split)[0] + + @property + def name(self): + """The final path component, if any.""" + return self.parser.split(str(self))[1] + + @property + def suffix(self): + """ + The final component's last suffix, if any. + + This includes the leading period. For example: '.txt' + """ + return self.parser.splitext(self.name)[1] + + @property + def suffixes(self): + """ + A list of the final component's suffixes, if any. + + These include the leading periods. For example: ['.tar', '.gz'] + """ + split = self.parser.splitext + stem, suffix = split(self.name) + suffixes = [] + while suffix: + suffixes.append(suffix) + stem, suffix = split(stem) + return suffixes[::-1] + + @property + def stem(self): + """The final path component, minus its last suffix.""" + return self.parser.splitext(self.name)[0] + + def with_name(self, name): + """Return a new path with the file name changed.""" + split = self.parser.split + if split(name)[0]: + raise ValueError(f"Invalid name {name!r}") + path = str(self) + path = path.removesuffix(split(path)[1]) + name + return self.with_segments(path) + + def with_stem(self, stem): + """Return a new path with the stem changed.""" + suffix = self.suffix + if not suffix: + return self.with_name(stem) + elif not stem: + # If the suffix is non-empty, we can't make the stem empty. + raise ValueError(f"{self!r} has a non-empty suffix") + else: + return self.with_name(stem + suffix) + + def with_suffix(self, suffix): + """Return a new path with the file suffix changed. If the path + has no suffix, add given suffix. If the given suffix is an empty + string, remove the suffix from the path. + """ + stem = self.stem + if not stem: + # If the stem is empty, we can't make the suffix non-empty. + raise ValueError(f"{self!r} has an empty name") + elif suffix and not suffix.startswith('.'): + raise ValueError(f"Invalid suffix {suffix!r}") + else: + return self.with_name(stem + suffix) + + @property + def parts(self): + """An object providing sequence-like access to the + components in the filesystem path.""" + anchor, parts = _explode_path(str(self), self.parser.split) + if anchor: + parts.append(anchor) + return tuple(reversed(parts)) + + def joinpath(self, *pathsegments): + """Combine this path with one or several arguments, and return a + new path representing either a subpath (if all arguments are relative + paths) or a totally different path (if one of the arguments is + anchored). + """ + return self.with_segments(str(self), *pathsegments) + + def __truediv__(self, key): + try: + return self.with_segments(str(self), key) + except TypeError: + return NotImplemented + + def __rtruediv__(self, key): + try: + return self.with_segments(key, str(self)) + except TypeError: + return NotImplemented + + @property + def parent(self): + """The logical parent of the path.""" + path = str(self) + parent = self.parser.split(path)[0] + if path != parent: + return self.with_segments(parent) + return self + + @property + def parents(self): + """A sequence of this path's logical parents.""" + split = self.parser.split + path = str(self) + parent = split(path)[0] + parents = [] + while path != parent: + parents.append(self.with_segments(parent)) + path = parent + parent = split(path)[0] + return tuple(parents) + + def full_match(self, pattern): + """ + Return True if this path matches the given glob-style pattern. The + pattern is matched against the entire path. + """ + case_sensitive = self.parser.normcase('Aa') == 'Aa' + globber = _PathGlobber(self.parser.sep, case_sensitive, recursive=True) + match = globber.compile(pattern, altsep=self.parser.altsep) + return match(str(self)) is not None + + +class _ReadablePath(_JoinablePath): + """Abstract base class for readable path objects. + + The Path class implements this ABC for local filesystem paths. Users may + create subclasses to implement readable virtual filesystem paths, such as + paths in archive files or on remote storage systems. + """ + __slots__ = () + + @property + @abstractmethod + def info(self): + """ + A PathInfo object that exposes the file type and other file attributes + of this path. + """ + raise NotImplementedError + + @abstractmethod + def __open_rb__(self, buffering=-1): + """ + Open the file pointed to by this path for reading in binary mode and + return a file object, like open(mode='rb'). + """ + raise NotImplementedError + + def read_bytes(self): + """ + Open the file in bytes mode, read it, and close the file. + """ + with magic_open(self, mode='rb', buffering=0) as f: + return f.read() + + def read_text(self, encoding=None, errors=None, newline=None): + """ + Open the file in text mode, read it, and close the file. + """ + # Call io.text_encoding() here to ensure any warning is raised at an + # appropriate stack level. + encoding = text_encoding(encoding) + with magic_open(self, mode='r', encoding=encoding, errors=errors, newline=newline) as f: + return f.read() + + @abstractmethod + def iterdir(self): + """Yield path objects of the directory contents. + + The children are yielded in arbitrary order, and the + special entries '.' and '..' are not included. + """ + raise NotImplementedError + + def glob(self, pattern, *, recurse_symlinks=True): + """Iterate over this subtree and yield all existing files (of any + kind, including directories) matching the given relative pattern. + """ + anchor, parts = _explode_path(pattern, self.parser.split) + if anchor: + raise NotImplementedError("Non-relative patterns are unsupported") + elif not parts: + raise ValueError(f"Unacceptable pattern: {pattern!r}") + elif not recurse_symlinks: + raise NotImplementedError("recurse_symlinks=False is unsupported") + case_sensitive = self.parser.normcase('Aa') == 'Aa' + globber = _PathGlobber(self.parser.sep, case_sensitive, recursive=True) + select = globber.selector(parts) + return select(self.joinpath('')) + + def walk(self, top_down=True, on_error=None, follow_symlinks=False): + """Walk the directory tree from this directory, similar to os.walk().""" + paths = [self] + while paths: + path = paths.pop() + if isinstance(path, tuple): + yield path + continue + dirnames = [] + filenames = [] + if not top_down: + paths.append((path, dirnames, filenames)) + try: + for child in path.iterdir(): + if child.info.is_dir(follow_symlinks=follow_symlinks): + if not top_down: + paths.append(child) + dirnames.append(child.name) + else: + filenames.append(child.name) + except OSError as error: + if on_error is not None: + on_error(error) + if not top_down: + while not isinstance(paths.pop(), tuple): + pass + continue + if top_down: + yield path, dirnames, filenames + paths += [path.joinpath(d) for d in reversed(dirnames)] + + @abstractmethod + def readlink(self): + """ + Return the path to which the symbolic link points. + """ + raise NotImplementedError + + def copy(self, target, **kwargs): + """ + Recursively copy this file or directory tree to the given destination. + """ + ensure_distinct_paths(self, target) + target._copy_from(self, **kwargs) + return target.joinpath() # Empty join to ensure fresh metadata. + + def copy_into(self, target_dir, **kwargs): + """ + Copy this file or directory tree into the given existing directory. + """ + name = self.name + if not name: + raise ValueError(f"{self!r} has an empty name") + return self.copy(target_dir / name, **kwargs) + + +class _WritablePath(_JoinablePath): + """Abstract base class for writable path objects. + + The Path class implements this ABC for local filesystem paths. Users may + create subclasses to implement writable virtual filesystem paths, such as + paths in archive files or on remote storage systems. + """ + __slots__ = () + + @abstractmethod + def symlink_to(self, target, target_is_directory=False): + """ + Make this path a symlink pointing to the target path. + Note the order of arguments (link, target) is the reverse of os.symlink. + """ + raise NotImplementedError + + @abstractmethod + def mkdir(self): + """ + Create a new directory at this given path. + """ + raise NotImplementedError + + @abstractmethod + def __open_wb__(self, buffering=-1): + """ + Open the file pointed to by this path for writing in binary mode and + return a file object, like open(mode='wb'). + """ + raise NotImplementedError + + def write_bytes(self, data): + """ + Open the file in bytes mode, write to it, and close the file. + """ + # type-check for the buffer interface before truncating the file + view = memoryview(data) + with magic_open(self, mode='wb') as f: + return f.write(view) + + def write_text(self, data, encoding=None, errors=None, newline=None): + """ + Open the file in text mode, write to it, and close the file. + """ + # Call io.text_encoding() here to ensure any warning is raised at an + # appropriate stack level. + encoding = text_encoding(encoding) + if not isinstance(data, str): + raise TypeError('data must be str, not %s' % + data.__class__.__name__) + with magic_open(self, mode='w', encoding=encoding, errors=errors, newline=newline) as f: + return f.write(data) + + def _copy_from(self, source, follow_symlinks=True): + """ + Recursively copy the given path to this path. + """ + stack = [(source, self)] + while stack: + src, dst = stack.pop() + if not follow_symlinks and src.info.is_symlink(): + dst.symlink_to(str(src.readlink()), src.info.is_dir()) + elif src.info.is_dir(): + children = src.iterdir() + dst.mkdir() + for child in children: + stack.append((child, dst.joinpath(child.name))) + else: + ensure_different_files(src, dst) + with magic_open(src, 'rb') as source_f: + with magic_open(dst, 'wb') as target_f: + copyfileobj(source_f, target_f) + + +_JoinablePath.register(PurePath) +_ReadablePath.register(Path) +_WritablePath.register(Path) diff --git a/Lib/pdb.py b/Lib/pdb.py index bf503f1e73e..ec6cf06e58b 100755 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -185,6 +185,15 @@ def __init__(self, completekey='tab', stdin=None, stdout=None, skip=None, self.commands_bnum = None # The breakpoint number for which we are # defining a list + def set_trace(self, frame=None, *, commands=None): + if frame is None: + frame = sys._getframe().f_back + + if commands is not None: + self.rcLines.extend(commands) + + super().set_trace(frame) + def sigint_handler(self, signum, frame): if self.allow_kbdint: raise KeyboardInterrupt diff --git a/Lib/pickle.py b/Lib/pickle.py index 550f8675f2c..beaefae0479 100644 --- a/Lib/pickle.py +++ b/Lib/pickle.py @@ -26,12 +26,11 @@ from types import FunctionType from copyreg import dispatch_table from copyreg import _extension_registry, _inverted_registry, _extension_cache -from itertools import islice +from itertools import batched from functools import partial import sys from sys import maxsize from struct import pack, unpack -import re import io import codecs import _compat_pickle @@ -51,7 +50,7 @@ bytes_types = (bytes, bytearray) # These are purely informational; no code uses these. -format_version = "4.0" # File format version we write +format_version = "5.0" # File format version we write compatible_formats = ["1.0", # Original protocol 0 "1.1", # Protocol 0 with INST added "1.2", # Original protocol 1 @@ -68,7 +67,7 @@ # The protocol we write by default. May be less than HIGHEST_PROTOCOL. # Only bump this if the oldest still supported version of Python already # includes it. -DEFAULT_PROTOCOL = 4 +DEFAULT_PROTOCOL = 5 class PickleError(Exception): """A common base class for the other pickling exceptions.""" @@ -188,7 +187,7 @@ def __init__(self, value): NEXT_BUFFER = b'\x97' # push next out-of-band buffer READONLY_BUFFER = b'\x98' # make top of stack readonly -__all__.extend([x for x in dir() if re.match("[A-Z][A-Z0-9_]+$", x)]) +__all__.extend(x for x in dir() if x.isupper() and not x.startswith('_')) class _Framer: @@ -313,38 +312,46 @@ def load_frame(self, frame_size): # Tools used for pickling. -def _getattribute(obj, name): - top = obj - for subpath in name.split('.'): - if subpath == '': - raise AttributeError("Can't get local attribute {!r} on {!r}" - .format(name, top)) - try: - parent = obj - obj = getattr(obj, subpath) - except AttributeError: - raise AttributeError("Can't get attribute {!r} on {!r}" - .format(name, top)) from None - return obj, parent +def _getattribute(obj, dotted_path): + for subpath in dotted_path: + obj = getattr(obj, subpath) + return obj def whichmodule(obj, name): """Find the module an object belong to.""" + dotted_path = name.split('.') module_name = getattr(obj, '__module__', None) - if module_name is not None: - return module_name - # Protect the iteration by using a list copy of sys.modules against dynamic - # modules that trigger imports of other modules upon calls to getattr. - for module_name, module in sys.modules.copy().items(): - if (module_name == '__main__' - or module_name == '__mp_main__' # bpo-42406 - or module is None): - continue - try: - if _getattribute(module, name)[0] is obj: - return module_name - except AttributeError: - pass - return '__main__' + if '' in dotted_path: + raise PicklingError(f"Can't pickle local object {obj!r}") + if module_name is None: + # Protect the iteration by using a list copy of sys.modules against dynamic + # modules that trigger imports of other modules upon calls to getattr. + for module_name, module in sys.modules.copy().items(): + if (module_name == '__main__' + or module_name == '__mp_main__' # bpo-42406 + or module is None): + continue + try: + if _getattribute(module, dotted_path) is obj: + return module_name + except AttributeError: + pass + module_name = '__main__' + + try: + __import__(module_name, level=0) + module = sys.modules[module_name] + except (ImportError, ValueError, KeyError) as exc: + raise PicklingError(f"Can't pickle {obj!r}: {exc!s}") + try: + if _getattribute(module, dotted_path) is obj: + return module_name + except AttributeError: + raise PicklingError(f"Can't pickle {obj!r}: " + f"it's not found as {module_name}.{name}") + + raise PicklingError( + f"Can't pickle {obj!r}: it's not the same object as {module_name}.{name}") def encode_long(x): r"""Encode a long to a two's complement little-endian binary string. @@ -396,6 +403,13 @@ def decode_long(data): """ return int.from_bytes(data, byteorder='little', signed=True) +def _T(obj): + cls = type(obj) + module = cls.__module__ + if module in (None, 'builtins', '__main__'): + return cls.__qualname__ + return f'{module}.{cls.__qualname__}' + _NoValue = object() @@ -409,7 +423,7 @@ def __init__(self, file, protocol=None, *, fix_imports=True, The optional *protocol* argument tells the pickler to use the given protocol; supported protocols are 0, 1, 2, 3, 4 and 5. - The default protocol is 4. It was introduced in Python 3.4, and + The default protocol is 5. It was introduced in Python 3.8, and is incompatible with previous versions. Specifying a negative protocol version selects the highest @@ -579,26 +593,29 @@ def save(self, obj, save_persistent_id=True): if reduce is not _NoValue: rv = reduce() else: - raise PicklingError("Can't pickle %r object: %r" % - (t.__name__, obj)) + raise PicklingError(f"Can't pickle {_T(t)} object") # Check for string returned by reduce(), meaning "save as global" if isinstance(rv, str): self.save_global(obj, rv) return - # Assert that reduce() returned a tuple - if not isinstance(rv, tuple): - raise PicklingError("%s must return string or tuple" % reduce) - - # Assert that it returned an appropriately sized tuple - l = len(rv) - if not (2 <= l <= 6): - raise PicklingError("Tuple returned by %s must have " - "two to six elements" % reduce) - - # Save the reduce() output and finally memoize the object - self.save_reduce(obj=obj, *rv) + try: + # Assert that reduce() returned a tuple + if not isinstance(rv, tuple): + raise PicklingError(f'__reduce__ must return a string or tuple, not {_T(rv)}') + + # Assert that it returned an appropriately sized tuple + l = len(rv) + if not (2 <= l <= 6): + raise PicklingError("tuple returned by __reduce__ " + "must contain 2 through 6 elements") + + # Save the reduce() output and finally memoize the object + self.save_reduce(obj=obj, *rv) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} object') + raise def persistent_id(self, obj): # This exists so a subclass can override it @@ -620,10 +637,12 @@ def save_reduce(self, func, args, state=None, listitems=None, dictitems=None, state_setter=None, *, obj=None): # This API is called by some subclasses - if not isinstance(args, tuple): - raise PicklingError("args from save_reduce() must be a tuple") if not callable(func): - raise PicklingError("func from save_reduce() must be callable") + raise PicklingError(f"first item of the tuple returned by __reduce__ " + f"must be callable, not {_T(func)}") + if not isinstance(args, tuple): + raise PicklingError(f"second item of the tuple returned by __reduce__ " + f"must be a tuple, not {_T(args)}") save = self.save write = self.write @@ -632,19 +651,30 @@ def save_reduce(self, func, args, state=None, listitems=None, if self.proto >= 2 and func_name == "__newobj_ex__": cls, args, kwargs = args if not hasattr(cls, "__new__"): - raise PicklingError("args[0] from {} args has no __new__" - .format(func_name)) + raise PicklingError("first argument to __newobj_ex__() has no __new__") if obj is not None and cls is not obj.__class__: - raise PicklingError("args[0] from {} args has the wrong class" - .format(func_name)) + raise PicklingError(f"first argument to __newobj_ex__() " + f"must be {obj.__class__!r}, not {cls!r}") if self.proto >= 4: - save(cls) - save(args) - save(kwargs) + try: + save(cls) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} class') + raise + try: + save(args) + save(kwargs) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} __new__ arguments') + raise write(NEWOBJ_EX) else: func = partial(cls.__new__, cls, *args, **kwargs) - save(func) + try: + save(func) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} reconstructor') + raise save(()) write(REDUCE) elif self.proto >= 2 and func_name == "__newobj__": @@ -676,18 +706,33 @@ def save_reduce(self, func, args, state=None, listitems=None, # Python 2.2). cls = args[0] if not hasattr(cls, "__new__"): - raise PicklingError( - "args[0] from __newobj__ args has no __new__") + raise PicklingError("first argument to __newobj__() has no __new__") if obj is not None and cls is not obj.__class__: - raise PicklingError( - "args[0] from __newobj__ args has the wrong class") + raise PicklingError(f"first argument to __newobj__() " + f"must be {obj.__class__!r}, not {cls!r}") args = args[1:] - save(cls) - save(args) + try: + save(cls) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} class') + raise + try: + save(args) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} __new__ arguments') + raise write(NEWOBJ) else: - save(func) - save(args) + try: + save(func) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} reconstructor') + raise + try: + save(args) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} reconstructor arguments') + raise write(REDUCE) if obj is not None: @@ -705,23 +750,35 @@ def save_reduce(self, func, args, state=None, listitems=None, # items and dict items (as (key, value) tuples), or None. if listitems is not None: - self._batch_appends(listitems) + self._batch_appends(listitems, obj) if dictitems is not None: - self._batch_setitems(dictitems) + self._batch_setitems(dictitems, obj) if state is not None: if state_setter is None: - save(state) + try: + save(state) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} state') + raise write(BUILD) else: # If a state_setter is specified, call it instead of load_build # to update obj's with its previous state. # First, push state_setter and its tuple of expected arguments # (obj, state) onto the stack. - save(state_setter) + try: + save(state_setter) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} state setter') + raise save(obj) # simple BINGET opcode as obj is already memoized. - save(state) + try: + save(state) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} state') + raise write(TUPLE2) # Trigger a state_setter(obj, state) function call. write(REDUCE) @@ -901,8 +958,12 @@ def save_tuple(self, obj): save = self.save memo = self.memo if n <= 3 and self.proto >= 2: - for element in obj: - save(element) + for i, element in enumerate(obj): + try: + save(element) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} item {i}') + raise # Subtle. Same as in the big comment below. if id(obj) in memo: get = self.get(memo[id(obj)][0]) @@ -916,8 +977,12 @@ def save_tuple(self, obj): # has more than 3 elements. write = self.write write(MARK) - for element in obj: - save(element) + for i, element in enumerate(obj): + try: + save(element) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} item {i}') + raise if id(obj) in memo: # Subtle. d was not in memo when we entered save_tuple(), so @@ -947,38 +1012,47 @@ def save_list(self, obj): self.write(MARK + LIST) self.memoize(obj) - self._batch_appends(obj) + self._batch_appends(obj, obj) dispatch[list] = save_list _BATCHSIZE = 1000 - def _batch_appends(self, items): + def _batch_appends(self, items, obj): # Helper to batch up APPENDS sequences save = self.save write = self.write if not self.bin: - for x in items: - save(x) + for i, x in enumerate(items): + try: + save(x) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} item {i}') + raise write(APPEND) return - it = iter(items) - while True: - tmp = list(islice(it, self._BATCHSIZE)) - n = len(tmp) - if n > 1: + start = 0 + for batch in batched(items, self._BATCHSIZE): + batch_len = len(batch) + if batch_len != 1: write(MARK) - for x in tmp: - save(x) + for i, x in enumerate(batch, start): + try: + save(x) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} item {i}') + raise write(APPENDS) - elif n: - save(tmp[0]) + else: + try: + save(batch[0]) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} item {start}') + raise write(APPEND) - # else tmp is empty, and we're done - if n < self._BATCHSIZE: - return + start += batch_len def save_dict(self, obj): if self.bin: @@ -987,11 +1061,11 @@ def save_dict(self, obj): self.write(MARK + DICT) self.memoize(obj) - self._batch_setitems(obj.items()) + self._batch_setitems(obj.items(), obj) dispatch[dict] = save_dict - def _batch_setitems(self, items): + def _batch_setitems(self, items, obj): # Helper to batch up SETITEMS sequences; proto >= 1 only save = self.save write = self.write @@ -999,28 +1073,34 @@ def _batch_setitems(self, items): if not self.bin: for k, v in items: save(k) - save(v) + try: + save(v) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} item {k!r}') + raise write(SETITEM) return - it = iter(items) - while True: - tmp = list(islice(it, self._BATCHSIZE)) - n = len(tmp) - if n > 1: + for batch in batched(items, self._BATCHSIZE): + if len(batch) != 1: write(MARK) - for k, v in tmp: + for k, v in batch: save(k) - save(v) + try: + save(v) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} item {k!r}') + raise write(SETITEMS) - elif n: - k, v = tmp[0] + else: + k, v = batch[0] save(k) - save(v) + try: + save(v) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} item {k!r}') + raise write(SETITEM) - # else tmp is empty, and we're done - if n < self._BATCHSIZE: - return def save_set(self, obj): save = self.save @@ -1033,17 +1113,15 @@ def save_set(self, obj): write(EMPTY_SET) self.memoize(obj) - it = iter(obj) - while True: - batch = list(islice(it, self._BATCHSIZE)) - n = len(batch) - if n > 0: - write(MARK) + for batch in batched(obj, self._BATCHSIZE): + write(MARK) + try: for item in batch: save(item) - write(ADDITEMS) - if n < self._BATCHSIZE: - return + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} element') + raise + write(ADDITEMS) dispatch[set] = save_set def save_frozenset(self, obj): @@ -1055,8 +1133,12 @@ def save_frozenset(self, obj): return write(MARK) - for item in obj: - save(item) + try: + for item in obj: + save(item) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} element') + raise if id(obj) in self.memo: # If the object is already in the memo, this means it is @@ -1075,24 +1157,10 @@ def save_global(self, obj, name=None): if name is None: name = getattr(obj, '__qualname__', None) - if name is None: - name = obj.__name__ + if name is None: + name = obj.__name__ module_name = whichmodule(obj, name) - try: - __import__(module_name, level=0) - module = sys.modules[module_name] - obj2, parent = _getattribute(module, name) - except (ImportError, KeyError, AttributeError): - raise PicklingError( - "Can't pickle %r: it's not found as %s.%s" % - (obj, module_name, name)) from None - else: - if obj2 is not obj: - raise PicklingError( - "Can't pickle %r: it's not the same object as %s.%s" % - (obj, module_name, name)) - if self.proto >= 2: code = _extension_registry.get((module_name, name), _NoValue) if code is not _NoValue: @@ -1109,10 +1177,7 @@ def save_global(self, obj, name=None): else: write(EXT4 + pack("= 3. + if self.proto >= 4: self.save(module_name) self.save(name) @@ -1144,8 +1209,7 @@ def save_global(self, obj, name=None): def _save_toplevel_by_name(self, module_name, name): if self.proto >= 3: # Non-ASCII identifiers are supported only with protocols >= 3. - self.write(GLOBAL + bytes(module_name, "utf-8") + b'\n' + - bytes(name, "utf-8") + b'\n') + encoding = "utf-8" else: if self.fix_imports: r_name_mapping = _compat_pickle.REVERSE_NAME_MAPPING @@ -1154,13 +1218,19 @@ def _save_toplevel_by_name(self, module_name, name): module_name, name = r_name_mapping[(module_name, name)] elif module_name in r_import_mapping: module_name = r_import_mapping[module_name] - try: - self.write(GLOBAL + bytes(module_name, "ascii") + b'\n' + - bytes(name, "ascii") + b'\n') - except UnicodeEncodeError: - raise PicklingError( - "can't pickle global identifier '%s.%s' using " - "pickle protocol %i" % (module_name, name, self.proto)) from None + encoding = "ascii" + try: + self.write(GLOBAL + bytes(module_name, encoding) + b'\n') + except UnicodeEncodeError: + raise PicklingError( + f"can't pickle module identifier {module_name!r} using " + f"pickle protocol {self.proto}") + try: + self.write(bytes(name, encoding) + b'\n') + except UnicodeEncodeError: + raise PicklingError( + f"can't pickle global identifier {name!r} using " + f"pickle protocol {self.proto}") def save_type(self, obj): if obj is type(None): @@ -1316,7 +1386,7 @@ def load_int(self): elif data == TRUE[1:]: val = True else: - val = int(data, 0) + val = int(data) self.append(val) dispatch[INT[0]] = load_int @@ -1336,7 +1406,7 @@ def load_long(self): val = self.readline()[:-1] if val and val[-1] == b'L'[0]: val = val[:-1] - self.append(int(val, 0)) + self.append(int(val)) dispatch[LONG[0]] = load_long def load_long1(self): @@ -1620,8 +1690,13 @@ def find_class(self, module, name): elif module in _compat_pickle.IMPORT_MAPPING: module = _compat_pickle.IMPORT_MAPPING[module] __import__(module, level=0) - if self.proto >= 4: - return _getattribute(sys.modules[module], name)[0] + if self.proto >= 4 and '.' in name: + dotted_path = name.split('.') + try: + return _getattribute(sys.modules[module], dotted_path) + except AttributeError: + raise AttributeError( + f"Can't resolve path {name!r} on module {module!r}") else: return getattr(sys.modules[module], name) @@ -1831,36 +1906,26 @@ def _loads(s, /, *, fix_imports=True, encoding="ASCII", errors="strict", Pickler, Unpickler = _Pickler, _Unpickler dump, dumps, load, loads = _dump, _dumps, _load, _loads -# Doctest -def _test(): - import doctest - return doctest.testmod() -if __name__ == "__main__": +def _main(args=None): import argparse + import pprint parser = argparse.ArgumentParser( - description='display contents of the pickle files') + description='display contents of the pickle files', + color=True, + ) parser.add_argument( 'pickle_file', - nargs='*', help='the pickle file') - parser.add_argument( - '-t', '--test', action='store_true', - help='run self-test suite') - parser.add_argument( - '-v', action='store_true', - help='run verbosely; only affects self-test run') - args = parser.parse_args() - if args.test: - _test() - else: - if not args.pickle_file: - parser.print_help() + nargs='+', help='the pickle file') + args = parser.parse_args(args) + for fn in args.pickle_file: + if fn == '-': + obj = load(sys.stdin.buffer) else: - import pprint - for fn in args.pickle_file: - if fn == '-': - obj = load(sys.stdin.buffer) - else: - with open(fn, 'rb') as f: - obj = load(f) - pprint.pprint(obj) + with open(fn, 'rb') as f: + obj = load(f) + pprint.pprint(obj) + + +if __name__ == "__main__": + _main() diff --git a/Lib/pickletools.py b/Lib/pickletools.py index 33a51492ea9..254b6c7fcc9 100644 --- a/Lib/pickletools.py +++ b/Lib/pickletools.py @@ -348,7 +348,7 @@ def read_stringnl(f, decode=True, stripquotes=True, *, encoding='latin-1'): for q in (b'"', b"'"): if data.startswith(q): if not data.endswith(q): - raise ValueError("strinq quote %r not found at both " + raise ValueError("string quote %r not found at both " "ends of %r" % (q, data)) data = data[1:-1] break @@ -2429,8 +2429,6 @@ def dis(pickle, out=None, memo=None, indentlevel=4, annotate=0): + A memo entry isn't referenced before it's defined. + The markobject isn't stored in the memo. - - + A memo entry isn't redefined. """ # Most of the hair here is for sanity checks, but most of it is needed @@ -2484,7 +2482,7 @@ def dis(pickle, out=None, memo=None, indentlevel=4, annotate=0): assert opcode.name == "POP" numtopop = 0 else: - errormsg = markmsg = "no MARK exists on stack" + errormsg = "no MARK exists on stack" # Check for correct memo usage. if opcode.name in ("PUT", "BINPUT", "LONG_BINPUT", "MEMOIZE"): @@ -2494,9 +2492,7 @@ def dis(pickle, out=None, memo=None, indentlevel=4, annotate=0): else: assert arg is not None memo_idx = arg - if memo_idx in memo: - errormsg = "memo key %r already defined" % arg - elif not stack: + if not stack: errormsg = "stack is empty -- can't store into memo" elif stack[-1] is markobject: errormsg = "can't store markobject in the memo" @@ -2842,17 +2838,16 @@ def __init__(self, value): 'disassembler_memo_test': _memo_test, } -def _test(): - import doctest - return doctest.testmod() if __name__ == "__main__": import argparse parser = argparse.ArgumentParser( - description='disassemble one or more pickle files') + description='disassemble one or more pickle files', + color=True, + ) parser.add_argument( 'pickle_file', - nargs='*', help='the pickle file') + nargs='+', help='the pickle file') parser.add_argument( '-o', '--output', help='the file where the output should be written') @@ -2869,36 +2864,24 @@ def _test(): '-p', '--preamble', default="==> {name} <==", help='if more than one pickle file is specified, print this before' ' each disassembly') - parser.add_argument( - '-t', '--test', action='store_true', - help='run self-test suite') - parser.add_argument( - '-v', action='store_true', - help='run verbosely; only affects self-test run') args = parser.parse_args() - if args.test: - _test() + annotate = 30 if args.annotate else 0 + memo = {} if args.memo else None + if args.output is None: + output = sys.stdout else: - if not args.pickle_file: - parser.print_help() - else: - annotate = 30 if args.annotate else 0 - memo = {} if args.memo else None - if args.output is None: - output = sys.stdout + output = open(args.output, 'w') + try: + for arg in args.pickle_file: + if len(args.pickle_file) > 1: + name = '' if arg == '-' else arg + preamble = args.preamble.format(name=name) + output.write(preamble + '\n') + if arg == '-': + dis(sys.stdin.buffer, output, memo, args.indentlevel, annotate) else: - output = open(args.output, 'w') - try: - for arg in args.pickle_file: - if len(args.pickle_file) > 1: - name = '' if arg == '-' else arg - preamble = args.preamble.format(name=name) - output.write(preamble + '\n') - if arg == '-': - dis(sys.stdin.buffer, output, memo, args.indentlevel, annotate) - else: - with open(arg, 'rb') as f: - dis(f, output, memo, args.indentlevel, annotate) - finally: - if output is not sys.stdout: - output.close() + with open(arg, 'rb') as f: + dis(f, output, memo, args.indentlevel, annotate) + finally: + if output is not sys.stdout: + output.close() diff --git a/Lib/pkgutil.py b/Lib/pkgutil.py index dccbec52aa7..8772a66791a 100644 --- a/Lib/pkgutil.py +++ b/Lib/pkgutil.py @@ -8,11 +8,9 @@ import os import os.path import sys -from types import ModuleType -import warnings __all__ = [ - 'get_importer', 'iter_importers', 'get_loader', 'find_loader', + 'get_importer', 'iter_importers', 'walk_packages', 'iter_modules', 'get_data', 'read_code', 'extend_path', 'ModuleInfo', @@ -263,59 +261,6 @@ def iter_importers(fullname=""): yield get_importer(item) -def get_loader(module_or_name): - """Get a "loader" object for module_or_name - - Returns None if the module cannot be found or imported. - If the named module is not already imported, its containing package - (if any) is imported, in order to establish the package __path__. - """ - warnings._deprecated("pkgutil.get_loader", - f"{warnings._DEPRECATED_MSG}; " - "use importlib.util.find_spec() instead", - remove=(3, 14)) - if module_or_name in sys.modules: - module_or_name = sys.modules[module_or_name] - if module_or_name is None: - return None - if isinstance(module_or_name, ModuleType): - module = module_or_name - loader = getattr(module, '__loader__', None) - if loader is not None: - return loader - if getattr(module, '__spec__', None) is None: - return None - fullname = module.__name__ - else: - fullname = module_or_name - return find_loader(fullname) - - -def find_loader(fullname): - """Find a "loader" object for fullname - - This is a backwards compatibility wrapper around - importlib.util.find_spec that converts most failures to ImportError - and only returns the loader rather than the full spec - """ - warnings._deprecated("pkgutil.find_loader", - f"{warnings._DEPRECATED_MSG}; " - "use importlib.util.find_spec() instead", - remove=(3, 14)) - if fullname.startswith('.'): - msg = "Relative module name {!r} not supported".format(fullname) - raise ImportError(msg) - try: - spec = importlib.util.find_spec(fullname) - except (ImportError, AttributeError, TypeError, ValueError) as ex: - # This hack fixes an impedance mismatch between pkgutil and - # importlib, where the latter raises other errors for cases where - # pkgutil previously raised ImportError - msg = "Error while finding loader for {!r} ({}: {})" - raise ImportError(msg.format(fullname, type(ex), ex)) from ex - return spec.loader if spec is not None else None - - def extend_path(path, name): """Extend a package's path. diff --git a/Lib/poplib.py b/Lib/poplib.py new file mode 100644 index 00000000000..4469bff44b4 --- /dev/null +++ b/Lib/poplib.py @@ -0,0 +1,477 @@ +"""A POP3 client class. + +Based on the J. Myers POP3 draft, Jan. 96 +""" + +# Author: David Ascher +# [heavily stealing from nntplib.py] +# Updated: Piers Lauder [Jul '97] +# String method conversion and test jig improvements by ESR, February 2001. +# Added the POP3_SSL class. Methods loosely based on IMAP_SSL. Hector Urtubia Aug 2003 + +# Example (see the test function at the end of this file) + +# Imports + +import errno +import re +import socket +import sys + +try: + import ssl + HAVE_SSL = True +except ImportError: + HAVE_SSL = False + +__all__ = ["POP3","error_proto"] + +# Exception raised when an error or invalid response is received: + +class error_proto(Exception): pass + +# Standard Port +POP3_PORT = 110 + +# POP SSL PORT +POP3_SSL_PORT = 995 + +# Line terminators (we always output CRLF, but accept any of CRLF, LFCR, LF) +CR = b'\r' +LF = b'\n' +CRLF = CR+LF + +# maximal line length when calling readline(). This is to prevent +# reading arbitrary length lines. RFC 1939 limits POP3 line length to +# 512 characters, including CRLF. We have selected 2048 just to be on +# the safe side. +_MAXLINE = 2048 + + +class POP3: + + """This class supports both the minimal and optional command sets. + Arguments can be strings or integers (where appropriate) + (e.g.: retr(1) and retr('1') both work equally well. + + Minimal Command Set: + USER name user(name) + PASS string pass_(string) + STAT stat() + LIST [msg] list(msg = None) + RETR msg retr(msg) + DELE msg dele(msg) + NOOP noop() + RSET rset() + QUIT quit() + + Optional Commands (some servers support these): + RPOP name rpop(name) + APOP name digest apop(name, digest) + TOP msg n top(msg, n) + UIDL [msg] uidl(msg = None) + CAPA capa() + STLS stls() + UTF8 utf8() + + Raises one exception: 'error_proto'. + + Instantiate with: + POP3(hostname, port=110) + + NB: the POP protocol locks the mailbox from user + authorization until QUIT, so be sure to get in, suck + the messages, and quit, each time you access the + mailbox. + + POP is a line-based protocol, which means large mail + messages consume lots of python cycles reading them + line-by-line. + + If it's available on your mail server, use IMAP4 + instead, it doesn't suffer from the two problems + above. + """ + + encoding = 'UTF-8' + + def __init__(self, host, port=POP3_PORT, + timeout=socket._GLOBAL_DEFAULT_TIMEOUT): + self.host = host + self.port = port + self._tls_established = False + sys.audit("poplib.connect", self, host, port) + self.sock = self._create_socket(timeout) + self.file = self.sock.makefile('rb') + self._debugging = 0 + self.welcome = self._getresp() + + def _create_socket(self, timeout): + if timeout is not None and not timeout: + raise ValueError('Non-blocking socket (timeout=0) is not supported') + return socket.create_connection((self.host, self.port), timeout) + + def _putline(self, line): + if self._debugging > 1: print('*put*', repr(line)) + sys.audit("poplib.putline", self, line) + self.sock.sendall(line + CRLF) + + + # Internal: send one command to the server (through _putline()) + + def _putcmd(self, line): + if self._debugging: print('*cmd*', repr(line)) + line = bytes(line, self.encoding) + self._putline(line) + + + # Internal: return one line from the server, stripping CRLF. + # This is where all the CPU time of this module is consumed. + # Raise error_proto('-ERR EOF') if the connection is closed. + + def _getline(self): + line = self.file.readline(_MAXLINE + 1) + if len(line) > _MAXLINE: + raise error_proto('line too long') + + if self._debugging > 1: print('*get*', repr(line)) + if not line: raise error_proto('-ERR EOF') + octets = len(line) + # server can send any combination of CR & LF + # however, 'readline()' returns lines ending in LF + # so only possibilities are ...LF, ...CRLF, CR...LF + if line[-2:] == CRLF: + return line[:-2], octets + if line[:1] == CR: + return line[1:-1], octets + return line[:-1], octets + + + # Internal: get a response from the server. + # Raise 'error_proto' if the response doesn't start with '+'. + + def _getresp(self): + resp, o = self._getline() + if self._debugging > 1: print('*resp*', repr(resp)) + if not resp.startswith(b'+'): + raise error_proto(resp) + return resp + + + # Internal: get a response plus following text from the server. + + def _getlongresp(self): + resp = self._getresp() + list = []; octets = 0 + line, o = self._getline() + while line != b'.': + if line.startswith(b'..'): + o = o-1 + line = line[1:] + octets = octets + o + list.append(line) + line, o = self._getline() + return resp, list, octets + + + # Internal: send a command and get the response + + def _shortcmd(self, line): + self._putcmd(line) + return self._getresp() + + + # Internal: send a command and get the response plus following text + + def _longcmd(self, line): + self._putcmd(line) + return self._getlongresp() + + + # These can be useful: + + def getwelcome(self): + return self.welcome + + + def set_debuglevel(self, level): + self._debugging = level + + + # Here are all the POP commands: + + def user(self, user): + """Send user name, return response + + (should indicate password required). + """ + return self._shortcmd('USER %s' % user) + + + def pass_(self, pswd): + """Send password, return response + + (response includes message count, mailbox size). + + NB: mailbox is locked by server from here to 'quit()' + """ + return self._shortcmd('PASS %s' % pswd) + + + def stat(self): + """Get mailbox status. + + Result is tuple of 2 ints (message count, mailbox size) + """ + retval = self._shortcmd('STAT') + rets = retval.split() + if self._debugging: print('*stat*', repr(rets)) + + # Check if the response has enough elements + # RFC 1939 requires at least 3 elements (+OK, message count, mailbox size) + # but allows additional data after the required fields + if len(rets) < 3: + raise error_proto("Invalid STAT response format") + + try: + numMessages = int(rets[1]) + sizeMessages = int(rets[2]) + except ValueError: + raise error_proto("Invalid STAT response data: non-numeric values") + + return (numMessages, sizeMessages) + + + def list(self, which=None): + """Request listing, return result. + + Result without a message number argument is in form + ['response', ['mesg_num octets', ...], octets]. + + Result when a message number argument is given is a + single response: the "scan listing" for that message. + """ + if which is not None: + return self._shortcmd('LIST %s' % which) + return self._longcmd('LIST') + + + def retr(self, which): + """Retrieve whole message number 'which'. + + Result is in form ['response', ['line', ...], octets]. + """ + return self._longcmd('RETR %s' % which) + + + def dele(self, which): + """Delete message number 'which'. + + Result is 'response'. + """ + return self._shortcmd('DELE %s' % which) + + + def noop(self): + """Does nothing. + + One supposes the response indicates the server is alive. + """ + return self._shortcmd('NOOP') + + + def rset(self): + """Unmark all messages marked for deletion.""" + return self._shortcmd('RSET') + + + def quit(self): + """Signoff: commit changes on server, unlock mailbox, close connection.""" + resp = self._shortcmd('QUIT') + self.close() + return resp + + def close(self): + """Close the connection without assuming anything about it.""" + try: + file = self.file + self.file = None + if file is not None: + file.close() + finally: + sock = self.sock + self.sock = None + if sock is not None: + try: + sock.shutdown(socket.SHUT_RDWR) + except OSError as exc: + # The server might already have closed the connection. + # On Windows, this may result in WSAEINVAL (error 10022): + # An invalid operation was attempted. + if (exc.errno != errno.ENOTCONN + and getattr(exc, 'winerror', 0) != 10022): + raise + finally: + sock.close() + + #__del__ = quit + + + # optional commands: + + def rpop(self, user): + """Send RPOP command to access the mailbox with an alternate user.""" + return self._shortcmd('RPOP %s' % user) + + + timestamp = re.compile(br'\+OK.[^<]*(<.*>)') + + def apop(self, user, password): + """Authorisation + + - only possible if server has supplied a timestamp in initial greeting. + + Args: + user - mailbox user; + password - mailbox password. + + NB: mailbox is locked by server from here to 'quit()' + """ + secret = bytes(password, self.encoding) + m = self.timestamp.match(self.welcome) + if not m: + raise error_proto('-ERR APOP not supported by server') + import hashlib + digest = m.group(1)+secret + digest = hashlib.md5(digest).hexdigest() + return self._shortcmd('APOP %s %s' % (user, digest)) + + + def top(self, which, howmuch): + """Retrieve message header of message number 'which' + and first 'howmuch' lines of message body. + + Result is in form ['response', ['line', ...], octets]. + """ + return self._longcmd('TOP %s %s' % (which, howmuch)) + + + def uidl(self, which=None): + """Return message digest (unique id) list. + + If 'which', result contains unique id for that message + in the form 'response mesgnum uid', otherwise result is + the list ['response', ['mesgnum uid', ...], octets] + """ + if which is not None: + return self._shortcmd('UIDL %s' % which) + return self._longcmd('UIDL') + + + def utf8(self): + """Try to enter UTF-8 mode (see RFC 6856). Returns server response. + """ + return self._shortcmd('UTF8') + + + def capa(self): + """Return server capabilities (RFC 2449) as a dictionary + >>> c=poplib.POP3('localhost') + >>> c.capa() + {'IMPLEMENTATION': ['Cyrus', 'POP3', 'server', 'v2.2.12'], + 'TOP': [], 'LOGIN-DELAY': ['0'], 'AUTH-RESP-CODE': [], + 'EXPIRE': ['NEVER'], 'USER': [], 'STLS': [], 'PIPELINING': [], + 'UIDL': [], 'RESP-CODES': []} + >>> + + Really, according to RFC 2449, the cyrus folks should avoid + having the implementation split into multiple arguments... + """ + def _parsecap(line): + lst = line.decode('ascii').split() + return lst[0], lst[1:] + + caps = {} + try: + resp = self._longcmd('CAPA') + rawcaps = resp[1] + for capline in rawcaps: + capnm, capargs = _parsecap(capline) + caps[capnm] = capargs + except error_proto: + raise error_proto('-ERR CAPA not supported by server') + return caps + + + def stls(self, context=None): + """Start a TLS session on the active connection as specified in RFC 2595. + + context - a ssl.SSLContext + """ + if not HAVE_SSL: + raise error_proto('-ERR TLS support missing') + if self._tls_established: + raise error_proto('-ERR TLS session already established') + caps = self.capa() + if not 'STLS' in caps: + raise error_proto('-ERR STLS not supported by server') + if context is None: + context = ssl._create_stdlib_context() + resp = self._shortcmd('STLS') + self.sock = context.wrap_socket(self.sock, + server_hostname=self.host) + self.file = self.sock.makefile('rb') + self._tls_established = True + return resp + + +if HAVE_SSL: + + class POP3_SSL(POP3): + """POP3 client class over SSL connection + + Instantiate with: POP3_SSL(hostname, port=995, context=None) + + hostname - the hostname of the pop3 over ssl server + port - port number + context - a ssl.SSLContext + + See the methods of the parent class POP3 for more documentation. + """ + + def __init__(self, host, port=POP3_SSL_PORT, + *, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, context=None): + if context is None: + context = ssl._create_stdlib_context() + self.context = context + POP3.__init__(self, host, port, timeout) + + def _create_socket(self, timeout): + sock = POP3._create_socket(self, timeout) + sock = self.context.wrap_socket(sock, + server_hostname=self.host) + return sock + + def stls(self, context=None): + """The method unconditionally raises an exception since the + STLS command doesn't make any sense on an already established + SSL/TLS session. + """ + raise error_proto('-ERR TLS session already established') + + __all__.append("POP3_SSL") + +if __name__ == "__main__": + a = POP3(sys.argv[1]) + print(a.getwelcome()) + a.user(sys.argv[2]) + a.pass_(sys.argv[3]) + a.list() + (numMsgs, totalSize) = a.stat() + for i in range(1, numMsgs + 1): + (header, msg, octets) = a.retr(i) + print("Message %d:" % i) + for line in msg: + print(' ' + line) + print('-----------------------') + a.quit() diff --git a/Lib/posixpath.py b/Lib/posixpath.py index 80561ae7e52..ad86cc06c01 100644 --- a/Lib/posixpath.py +++ b/Lib/posixpath.py @@ -284,42 +284,41 @@ def expanduser(path): # This expands the forms $variable and ${variable} only. # Non-existent variables are left unchanged. -_varprog = None -_varprogb = None +_varpattern = r'\$(\w+|\{[^}]*\}?)' +_varsub = None +_varsubb = None def expandvars(path): """Expand shell variables of form $var and ${var}. Unknown variables are left unchanged.""" path = os.fspath(path) - global _varprog, _varprogb + global _varsub, _varsubb if isinstance(path, bytes): if b'$' not in path: return path - if not _varprogb: + if not _varsubb: import re - _varprogb = re.compile(br'\$(\w+|\{[^}]*\})', re.ASCII) - search = _varprogb.search + _varsubb = re.compile(_varpattern.encode(), re.ASCII).sub + sub = _varsubb start = b'{' end = b'}' environ = getattr(os, 'environb', None) else: if '$' not in path: return path - if not _varprog: + if not _varsub: import re - _varprog = re.compile(r'\$(\w+|\{[^}]*\})', re.ASCII) - search = _varprog.search + _varsub = re.compile(_varpattern, re.ASCII).sub + sub = _varsub start = '{' end = '}' environ = os.environ - i = 0 - while True: - m = search(path, i) - if not m: - break - i, j = m.span(0) - name = m.group(1) - if name.startswith(start) and name.endswith(end): + + def repl(m): + name = m[1] + if name.startswith(start): + if not name.endswith(end): + return m[0] name = name[1:-1] try: if environ is None: @@ -327,13 +326,11 @@ def expandvars(path): else: value = environ[name] except KeyError: - i = j + return m[0] else: - tail = path[j:] - path = path[:i] + value - i = len(path) - path += tail - return path + return value + + return sub(repl, path) # Normalize a path, e.g. A//B, A/./B and A/foo/../B all become A/B. @@ -410,6 +407,8 @@ def realpath(filename, *, strict=False): else: ignored_error = OSError + lstat = os.lstat + readlink = os.readlink maxlinks = None # The stack of unresolved path parts. When popped, a special value of None @@ -432,6 +431,10 @@ def realpath(filename, *, strict=False): # the same links. seen = {} + # Number of symlinks traversed. When the number of traversals is limited + # by *maxlinks*, this is used instead of *seen* to detect symlink loops. + link_count = 0 + while part_count: name = rest.pop() if name is None: @@ -451,14 +454,22 @@ def realpath(filename, *, strict=False): else: newpath = path + sep + name try: - st_mode = os.lstat(newpath).st_mode + st_mode = lstat(newpath).st_mode if not stat.S_ISLNK(st_mode): if strict and part_count and not stat.S_ISDIR(st_mode): raise OSError(errno.ENOTDIR, os.strerror(errno.ENOTDIR), newpath) path = newpath continue - if newpath in seen: + elif maxlinks is not None: + link_count += 1 + if link_count > maxlinks: + if strict: + raise OSError(errno.ELOOP, os.strerror(errno.ELOOP), + newpath) + path = newpath + continue + elif newpath in seen: # Already seen this path path = seen[newpath] if path is not None: @@ -466,11 +477,11 @@ def realpath(filename, *, strict=False): continue # The symlink is not resolved, so we must have a symlink loop. if strict: - # Raise OSError(errno.ELOOP) - os.stat(newpath) + raise OSError(errno.ELOOP, os.strerror(errno.ELOOP), + newpath) path = newpath continue - target = os.readlink(newpath) + target = readlink(newpath) except ignored_error: pass else: diff --git a/Lib/pprint.py b/Lib/pprint.py index 9314701db34..dc0953cec67 100644 --- a/Lib/pprint.py +++ b/Lib/pprint.py @@ -35,8 +35,6 @@ """ import collections as _collections -import dataclasses as _dataclasses -import re import sys as _sys import types as _types from io import StringIO as _StringIO @@ -54,6 +52,7 @@ def pprint(object, stream=None, indent=1, width=80, depth=None, *, underscore_numbers=underscore_numbers) printer.pprint(object) + def pformat(object, indent=1, width=80, depth=None, *, compact=False, sort_dicts=True, underscore_numbers=False): """Format a Python object into a pretty-printed representation.""" @@ -61,22 +60,27 @@ def pformat(object, indent=1, width=80, depth=None, *, compact=compact, sort_dicts=sort_dicts, underscore_numbers=underscore_numbers).pformat(object) + def pp(object, *args, sort_dicts=False, **kwargs): """Pretty-print a Python object""" pprint(object, *args, sort_dicts=sort_dicts, **kwargs) + def saferepr(object): """Version of repr() which can handle recursive data structures.""" return PrettyPrinter()._safe_repr(object, {}, None, 0)[0] + def isreadable(object): """Determine if saferepr(object) is readable by eval().""" return PrettyPrinter()._safe_repr(object, {}, None, 0)[1] + def isrecursive(object): """Determine if object requires a recursive representation.""" return PrettyPrinter()._safe_repr(object, {}, None, 0)[2] + class _safe_key: """Helper function for key functions when sorting unorderable objects. @@ -99,10 +103,12 @@ def __lt__(self, other): return ((str(type(self.obj)), id(self.obj)) < \ (str(type(other.obj)), id(other.obj))) + def _safe_tuple(t): "Helper function for comparing 2-tuples" return _safe_key(t[0]), _safe_key(t[1]) + class PrettyPrinter: def __init__(self, indent=1, width=80, depth=None, stream=None, *, compact=False, sort_dicts=True, underscore_numbers=False): @@ -179,12 +185,15 @@ def _format(self, object, stream, indent, allowance, context, level): max_width = self._width - indent - allowance if len(rep) > max_width: p = self._dispatch.get(type(object).__repr__, None) + # Lazy import to improve module import time + from dataclasses import is_dataclass + if p is not None: context[objid] = 1 p(self, object, stream, indent, allowance, context, level + 1) del context[objid] return - elif (_dataclasses.is_dataclass(object) and + elif (is_dataclass(object) and not isinstance(object, type) and object.__dataclass_params__.repr and # Check dataclass has generated repr method. @@ -197,9 +206,12 @@ def _format(self, object, stream, indent, allowance, context, level): stream.write(rep) def _pprint_dataclass(self, object, stream, indent, allowance, context, level): + # Lazy import to improve module import time + from dataclasses import fields as dataclass_fields + cls_name = object.__class__.__name__ indent += len(cls_name) + 1 - items = [(f.name, getattr(object, f.name)) for f in _dataclasses.fields(object) if f.repr] + items = [(f.name, getattr(object, f.name)) for f in dataclass_fields(object) if f.repr] stream.write(cls_name + '(') self._format_namespace_items(items, stream, indent, allowance, context, level) stream.write(')') @@ -291,6 +303,9 @@ def _pprint_str(self, object, stream, indent, allowance, context, level): if len(rep) <= max_width1: chunks.append(rep) else: + # Lazy import to improve module import time + import re + # A list of alternating (non-space, space) strings parts = re.findall(r'\S*\s*', line) assert parts @@ -632,9 +647,11 @@ def _safe_repr(self, object, context, maxlevels, level): rep = repr(object) return rep, (rep and not rep.startswith('<')), False + _builtin_scalars = frozenset({str, bytes, bytearray, float, complex, bool, type(None)}) + def _recursion(object): return ("" % (type(object).__name__, id(object))) diff --git a/Lib/py_compile.py b/Lib/py_compile.py index 388614e51b1..43d8ec90ffb 100644 --- a/Lib/py_compile.py +++ b/Lib/py_compile.py @@ -177,7 +177,7 @@ def main(): import argparse description = 'A simple command-line interface for py_compile module.' - parser = argparse.ArgumentParser(description=description) + parser = argparse.ArgumentParser(description=description, color=True) parser.add_argument( '-q', '--quiet', action='store_true', diff --git a/Lib/pydoc.py b/Lib/pydoc.py index b521a550472..1f8a6ef3d7c 100644 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """Generate Python documentation in HTML or text for interactive use. At the Python interactive prompt, calling help(thing) on a Python object @@ -16,12 +15,15 @@ class or function within a module or module in a package. If the Run "pydoc -k " to search for a keyword in the synopsis lines of all available modules. +Run "pydoc -n " to start an HTTP server with the given +hostname (default: localhost) on the local machine. + Run "pydoc -p " to start an HTTP server on the given port on the local machine. Port number 0 can be used to get an arbitrary unused port. Run "pydoc -b" to start an HTTP server on an arbitrary unused port and -open a Web browser to interactively browse documentation. The -p option -can be used with the -b option to explicitly specify the server port. +open a web browser to interactively browse documentation. Combine with +the -n and -p options to control the hostname and port used. Run "pydoc -w " to write out the HTML documentation for a module to a file named ".html". @@ -51,6 +53,8 @@ class or function within a module or module in a package. If the # the current directory is changed with os.chdir(), an incorrect # path will be displayed. +import ast +import __future__ import builtins import importlib._bootstrap import importlib._bootstrap_external @@ -63,14 +67,32 @@ class or function within a module or module in a package. If the import platform import re import sys +import sysconfig +import textwrap import time import tokenize import urllib.parse import warnings +from annotationlib import Format from collections import deque from reprlib import Repr from traceback import format_exception_only +from _pyrepl.pager import (get_pager, pipe_pager, + plain_pager, tempfile_pager, tty_pager) + +# Expose plain() as pydoc.plain() +from _pyrepl.pager import plain # noqa: F401 + + +# --------------------------------------------------------- old names + +getpager = get_pager +pipepager = pipe_pager +plainpager = plain_pager +tempfilepager = tempfile_pager +ttypager = tty_pager + # --------------------------------------------------------- common routines @@ -86,9 +108,100 @@ def pathdirs(): normdirs.append(normdir) return dirs +def _findclass(func): + cls = sys.modules.get(func.__module__) + if cls is None: + return None + for name in func.__qualname__.split('.')[:-1]: + cls = getattr(cls, name) + if not inspect.isclass(cls): + return None + return cls + +def _finddoc(obj): + if inspect.ismethod(obj): + name = obj.__func__.__name__ + self = obj.__self__ + if (inspect.isclass(self) and + getattr(getattr(self, name, None), '__func__') is obj.__func__): + # classmethod + cls = self + else: + cls = self.__class__ + elif inspect.isfunction(obj): + name = obj.__name__ + cls = _findclass(obj) + if cls is None or getattr(cls, name) is not obj: + return None + elif inspect.isbuiltin(obj): + name = obj.__name__ + self = obj.__self__ + if (inspect.isclass(self) and + self.__qualname__ + '.' + name == obj.__qualname__): + # classmethod + cls = self + else: + cls = self.__class__ + # Should be tested before isdatadescriptor(). + elif isinstance(obj, property): + name = obj.__name__ + cls = _findclass(obj.fget) + if cls is None or getattr(cls, name) is not obj: + return None + elif inspect.ismethoddescriptor(obj) or inspect.isdatadescriptor(obj): + name = obj.__name__ + cls = obj.__objclass__ + if getattr(cls, name) is not obj: + return None + if inspect.ismemberdescriptor(obj): + slots = getattr(cls, '__slots__', None) + if isinstance(slots, dict) and name in slots: + return slots[name] + else: + return None + for base in cls.__mro__: + try: + doc = _getowndoc(getattr(base, name)) + except AttributeError: + continue + if doc is not None: + return doc + return None + +def _getowndoc(obj): + """Get the documentation string for an object if it is not + inherited from its class.""" + try: + doc = object.__getattribute__(obj, '__doc__') + if doc is None: + return None + if obj is not type: + typedoc = type(obj).__doc__ + if isinstance(typedoc, str) and typedoc == doc: + return None + return doc + except AttributeError: + return None + +def _getdoc(object): + """Get the documentation string for an object. + + All tabs are expanded to spaces. To clean up docstrings that are + indented to line up with blocks of code, any whitespace than can be + uniformly removed from the second line onwards is removed.""" + doc = _getowndoc(object) + if doc is None: + try: + doc = _finddoc(object) + except (AttributeError, TypeError): + return None + if not isinstance(doc, str): + return None + return inspect.cleandoc(doc) + def getdoc(object): """Get the doc string or comments for an object.""" - result = inspect.getdoc(object) or inspect.getcomments(object) + result = _getdoc(object) or inspect.getcomments(object) return result and re.sub('^ *\n', '', result.rstrip()) or '' def splitdoc(doc): @@ -100,6 +213,27 @@ def splitdoc(doc): return lines[0], '\n'.join(lines[2:]) return '', '\n'.join(lines) +def _getargspec(object): + try: + signature = inspect.signature(object, annotation_format=Format.STRING) + if signature: + name = getattr(object, '__name__', '') + # function are always single-line and should not be formatted + max_width = (80 - len(name)) if name != '' else None + return signature.format(max_width=max_width, quote_annotation_strings=False) + except (ValueError, TypeError): + argspec = getattr(object, '__text_signature__', None) + if argspec: + if argspec[:2] == '($': + argspec = '(' + argspec[2:] + if getattr(object, '__self__', None) is not None: + # Strip the bound argument. + m = re.match(r'\(\w+(?:(?=\))|,\s*(?:/(?:(?=\))|,\s*))?)', argspec) + if m: + argspec = '(' + argspec[m.end():] + return argspec + return None + def classname(object, modname): """Get a class name and qualify it with a module name if necessary.""" name = object.__name__ @@ -107,6 +241,19 @@ def classname(object, modname): name = object.__module__ + '.' + name return name +def parentname(object, modname): + """Get a name of the enclosing class (qualified it with a module name + if necessary) or module.""" + if '.' in object.__qualname__: + name = object.__qualname__.rpartition('.')[0] + if object.__module__ != modname and object.__module__ is not None: + return object.__module__ + '.' + name + else: + return name + else: + if object.__module__ != modname: + return object.__module__ + def isdata(object): """Check if an object is of a type that probably means it's data.""" return not (inspect.ismodule(object) or inspect.isclass(object) or @@ -134,12 +281,6 @@ def stripid(text): # The behaviour of %p is implementation-dependent in terms of case. return _re_stripid.sub(r'\1', text) -def _is_some_method(obj): - return (inspect.isfunction(obj) or - inspect.ismethod(obj) or - inspect.isbuiltin(obj) or - inspect.ismethoddescriptor(obj)) - def _is_bound_method(fn): """ Returns True if fn is a bound method, regardless of whether @@ -155,7 +296,7 @@ def _is_bound_method(fn): def allmethods(cl): methods = {} - for key, value in inspect.getmembers(cl, _is_some_method): + for key, value in inspect.getmembers(cl, inspect.isroutine): methods[key] = 1 for base in cl.__bases__: methods.update(allmethods(base)) # all your base are belong to us @@ -180,6 +321,8 @@ def _split_list(s, predicate): no.append(x) return yes, no +_future_feature_names = set(__future__.all_feature_names) + def visiblename(name, all=None, obj=None): """Decide whether to show documentation on a variable.""" # Certain special names are redundant or internal. @@ -187,13 +330,19 @@ def visiblename(name, all=None, obj=None): if name in {'__author__', '__builtins__', '__cached__', '__credits__', '__date__', '__doc__', '__file__', '__spec__', '__loader__', '__module__', '__name__', '__package__', - '__path__', '__qualname__', '__slots__', '__version__'}: + '__path__', '__qualname__', '__slots__', '__version__', + '__static_attributes__', '__firstlineno__', + '__annotate_func__', '__annotations_cache__'}: return 0 # Private names are hidden, but special names are displayed. if name.startswith('__') and name.endswith('__'): return 1 # Namedtuples have public fields and methods with a single leading underscore if name.startswith('_') and hasattr(obj, '_fields'): return True + # Ignore __future__ imports. + if obj is not __future__ and name in _future_feature_names: + if isinstance(getattr(obj, name, None), __future__._Feature): + return False if all is not None: # only document that which the programmer exported in __all__ return name in all @@ -201,11 +350,15 @@ def visiblename(name, all=None, obj=None): return not name.startswith('_') def classify_class_attrs(object): - """Wrap inspect.classify_class_attrs, with fixup for data descriptors.""" + """Wrap inspect.classify_class_attrs, with fixup for data descriptors and bound methods.""" results = [] for (name, kind, cls, value) in inspect.classify_class_attrs(object): if inspect.isdatadescriptor(value): kind = 'data descriptor' + if isinstance(value, property) and value.fset is None: + kind = 'readonly property' + elif kind == 'method' and _is_bound_method(value): + kind = 'static method' results.append((name, kind, cls, value)) return results @@ -225,6 +378,8 @@ def sort_attributes(attrs, object): def ispackage(path): """Guess whether a path refers to a package directory.""" + warnings.warn('The pydoc.ispackage() function is deprecated', + DeprecationWarning, stacklevel=2) if os.path.isdir(path): for ext in ('.py', '.pyc'): if os.path.isfile(os.path.join(path, '__init__' + ext)): @@ -232,21 +387,29 @@ def ispackage(path): return False def source_synopsis(file): - line = file.readline() - while line[:1] == '#' or not line.strip(): - line = file.readline() - if not line: break - line = line.strip() - if line[:4] == 'r"""': line = line[1:] - if line[:3] == '"""': - line = line[3:] - if line[-1:] == '\\': line = line[:-1] - while not line.strip(): - line = file.readline() - if not line: break - result = line.split('"""')[0].strip() - else: result = None - return result + """Return the one-line summary of a file object, if present""" + + string = '' + try: + tokens = tokenize.generate_tokens(file.readline) + for tok_type, tok_string, _, _, _ in tokens: + if tok_type == tokenize.STRING: + string += tok_string + elif tok_type == tokenize.NEWLINE: + with warnings.catch_warnings(): + # Ignore the "invalid escape sequence" warning. + warnings.simplefilter("ignore", SyntaxWarning) + docstring = ast.literal_eval(string) + if not isinstance(docstring, str): + return None + return docstring.strip().split('\n')[0].strip() + elif tok_type == tokenize.OP and tok_string in ('(', ')'): + string += tok_string + elif tok_type not in (tokenize.COMMENT, tokenize.NL, tokenize.ENCODING): + return None + except (tokenize.TokenError, UnicodeDecodeError, SyntaxError): + return None + return None def synopsis(filename, cache={}): """Get the one-line summary out of a module file.""" @@ -290,8 +453,17 @@ def synopsis(filename, cache={}): class ErrorDuringImport(Exception): """Errors that occurred while trying to import something to document it.""" def __init__(self, filename, exc_info): + if not isinstance(exc_info, tuple): + assert isinstance(exc_info, BaseException) + self.exc = type(exc_info) + self.value = exc_info + self.tb = exc_info.__traceback__ + else: + warnings.warn("A tuple value for exc_info is deprecated, use an exception instance", + DeprecationWarning) + + self.exc, self.value, self.tb = exc_info self.filename = filename - self.exc, self.value, self.tb = exc_info def __str__(self): exc = self.exc.__name__ @@ -312,8 +484,8 @@ def importfile(path): spec = importlib.util.spec_from_file_location(name, path, loader=loader) try: return importlib._bootstrap._load(spec) - except: - raise ErrorDuringImport(path, sys.exc_info()) + except BaseException as err: + raise ErrorDuringImport(path, err) def safeimport(path, forceload=0, cache={}): """Import a module; handle errors; return None if the module isn't found. @@ -340,25 +512,21 @@ def safeimport(path, forceload=0, cache={}): # Prevent garbage collection. cache[key] = sys.modules[key] del sys.modules[key] - module = __import__(path) - except: + module = importlib.import_module(path) + except BaseException as err: # Did the error occur before or after the module was found? - (exc, value, tb) = info = sys.exc_info() if path in sys.modules: # An error occurred while executing the imported module. - raise ErrorDuringImport(sys.modules[path].__file__, info) - elif exc is SyntaxError: + raise ErrorDuringImport(sys.modules[path].__file__, err) + elif type(err) is SyntaxError: # A SyntaxError occurred before we could execute the module. - raise ErrorDuringImport(value.filename, info) - elif issubclass(exc, ImportError) and value.name == path: + raise ErrorDuringImport(err.filename, err) + elif isinstance(err, ImportError) and err.name == path: # No such module in the path. return None else: # Some other error occurred during the importing process. - raise ErrorDuringImport(path, sys.exc_info()) - for part in path.split('.')[1:]: - try: module = getattr(module, part) - except AttributeError: return None + raise ErrorDuringImport(path, err) return module # ---------------------------------------------------- formatter base class @@ -376,15 +544,13 @@ def document(self, object, name=None, *args): # identifies something in a way that pydoc itself has issues handling; # think 'super' and how it is a descriptor (which raises the exception # by lacking a __name__ attribute) and an instance. - if inspect.isgetsetdescriptor(object): return self.docdata(*args) - if inspect.ismemberdescriptor(object): return self.docdata(*args) try: if inspect.ismodule(object): return self.docmodule(*args) if inspect.isclass(object): return self.docclass(*args) if inspect.isroutine(object): return self.docroutine(*args) except AttributeError: pass - if isinstance(object, property): return self.docproperty(*args) + if inspect.isdatadescriptor(object): return self.docdata(*args) return self.docother(*args) def fail(self, object, name=None, *args): @@ -395,9 +561,7 @@ def fail(self, object, name=None, *args): docmodule = docclass = docroutine = docother = docproperty = docdata = fail - def getdocloc(self, object, - basedir=os.path.join(sys.base_exec_prefix, "lib", - "python%d.%d" % sys.version_info[:2])): + def getdocloc(self, object, basedir=sysconfig.get_path('stdlib')): """Return the location of module docs or None""" try: @@ -409,16 +573,26 @@ def getdocloc(self, object, basedir = os.path.normcase(basedir) if (isinstance(object, type(os)) and - (object.__name__ in ('errno', 'exceptions', 'gc', 'imp', + (object.__name__ in ('errno', 'exceptions', 'gc', 'marshal', 'posix', 'signal', 'sys', '_thread', 'zipimport') or (file.startswith(basedir) and not file.startswith(os.path.join(basedir, 'site-packages')))) and - object.__name__ not in ('xml.etree', 'test.pydoc_mod')): - if docloc.startswith(("http://", "https://")): - docloc = "%s/%s" % (docloc.rstrip("/"), object.__name__.lower()) + object.__name__ not in ('xml.etree', 'test.test_pydoc.pydoc_mod')): + + try: + from pydoc_data import module_docs + except ImportError: + module_docs = None + + if module_docs and object.__name__ in module_docs.module_docs: + doc_name = module_docs.module_docs[object.__name__] + if docloc.startswith(("http://", "https://")): + docloc = "{}/{}".format(docloc.rstrip("/"), doc_name) + else: + docloc = os.path.join(docloc, doc_name) else: - docloc = os.path.join(docloc, object.__name__.lower() + ".html") + docloc = None else: docloc = None return docloc @@ -454,7 +628,7 @@ def repr_string(self, x, level): # needed to make any special characters, so show a raw string. return 'r' + testrepr[0] + self.escape(test) + testrepr[0] return re.sub(r'((\\[\\abfnrtv\'"]|\\[0-9]..|\\x..|\\u....)+)', - r'\1', + r'\1', self.escape(testrepr)) repr_str = repr_string @@ -479,49 +653,48 @@ class HTMLDoc(Doc): def page(self, title, contents): """Format an HTML page.""" return '''\ - -Python: %s - - + + + + +Python: %s + %s ''' % (title, contents) - def heading(self, title, fgcol, bgcol, extras=''): + def heading(self, title, extras=''): """Format a page heading.""" return ''' -
- -
 
- 
%s
%s
- ''' % (bgcol, fgcol, title, fgcol, extras or ' ') - - def section(self, title, fgcol, bgcol, contents, width=6, + + + +
 
%s
%s
+ ''' % (title, extras or ' ') + + def section(self, title, cls, contents, width=6, prelude='', marginalia=None, gap=' '): """Format a section with a heading.""" if marginalia is None: - marginalia = '' + ' ' * width + '' + marginalia = '' + ' ' * width + '' result = '''

- - - - ''' % (bgcol, fgcol, title) +
 
-%s
+ + + ''' % (cls, title) if prelude: result = result + ''' - - -''' % (bgcol, marginalia, prelude, gap) + + +''' % (cls, marginalia, cls, prelude, gap) else: result = result + ''' -''' % (bgcol, marginalia, gap) +''' % (cls, marginalia, gap) - return result + '\n
 
%s
%s%s
%s
%s%s
%s
%s%s
%s%s%s
' % contents + return result + '\n%s' % contents def bigsection(self, title, *args): """Format a section with a big heading.""" - title = '%s' % title + title = '%s' % title return self.section(title, *args) def preformat(self, text): @@ -530,19 +703,19 @@ def preformat(self, text): return replace(text, '\n\n', '\n \n', '\n\n', '\n \n', ' ', ' ', '\n', '
\n') - def multicolumn(self, list, format, cols=4): + def multicolumn(self, list, format): """Format a list of items into a multi-column list.""" result = '' - rows = (len(list)+cols-1)//cols - for col in range(cols): - result = result + '' % (100//cols) + rows = (len(list) + 3) // 4 + for col in range(4): + result = result + '' for i in range(rows*col, rows*col+rows): if i < len(list): result = result + format(list[i]) + '
\n' result = result + '' - return '%s
' % result + return '%s
' % result - def grey(self, text): return '%s' % text + def grey(self, text): return '%s' % text def namelink(self, name, *dicts): """Make a link for an identifier, given name-to-URL mappings.""" @@ -559,6 +732,25 @@ def classlink(self, object, modname): module.__name__, name, classname(object, modname)) return classname(object, modname) + def parentlink(self, object, modname): + """Make a link for the enclosing class or module.""" + link = None + name, module = object.__name__, sys.modules.get(object.__module__) + if hasattr(module, name) and getattr(module, name) is object: + if '.' in object.__qualname__: + name = object.__qualname__.rpartition('.')[0] + if object.__module__ != modname: + link = '%s.html#%s' % (module.__name__, name) + else: + link = '#%s' % name + else: + if object.__module__ != modname: + link = '%s.html' % module.__name__ + if link: + return '%s' % (link, parentname(object, modname)) + else: + return parentname(object, modname) + def modulelink(self, object): """Make a link for a module.""" return '%s' % (object.__name__, object.__name__) @@ -588,13 +780,11 @@ def markup(self, text, escape=None, funcs={}, classes={}, methods={}): escape = escape or self.escape results = [] here = 0 - pattern = re.compile(r'\b((http|ftp)://\S+[\w/]|' + pattern = re.compile(r'\b((http|https|ftp)://\S+[\w/]|' r'RFC[- ]?(\d+)|' r'PEP[- ]?(\d+)|' r'(self\.)?(\w+))') - while True: - match = pattern.search(text, here) - if not match: break + while match := pattern.search(text, here): start, end = match.span() results.append(escape(text[here:start])) @@ -603,10 +793,10 @@ def markup(self, text, escape=None, funcs={}, classes={}, methods={}): url = escape(all).replace('"', '"') results.append('%s' % (url, url)) elif rfc: - url = 'http://www.rfc-editor.org/rfc/rfc%d.txt' % int(rfc) + url = 'https://www.rfc-editor.org/rfc/rfc%d.txt' % int(rfc) results.append('%s' % (url, escape(all))) elif pep: - url = 'http://www.python.org/dev/peps/pep-%04d/' % int(pep) + url = 'https://peps.python.org/pep-%04d/' % int(pep) results.append('%s' % (url, escape(all))) elif selfdot: # Create a link for methods like 'self.method(...)' @@ -629,17 +819,17 @@ def formattree(self, tree, modname, parent=None): """Produce HTML for a class tree as given by inspect.getclasstree().""" result = '' for entry in tree: - if type(entry) is type(()): + if isinstance(entry, tuple): c, bases = entry - result = result + '

' + result = result + '
' result = result + self.classlink(c, modname) if bases and bases != (parent,): parents = [] for base in bases: parents.append(self.classlink(base, modname)) result = result + '(' + ', '.join(parents) + ')' - result = result + '\n
' - elif type(entry) is type([]): + result = result + '\n' + elif isinstance(entry, list): result = result + '
\n%s
\n' % self.formattree( entry, modname, c) return '
\n%s
\n' % result @@ -655,10 +845,10 @@ def docmodule(self, object, name=None, mod=None, *ignored): links = [] for i in range(len(parts)-1): links.append( - '%s' % + '%s' % ('.'.join(parts[:i+1]), parts[i])) linkedname = '.'.join(links + parts[-1:]) - head = '%s' % linkedname + head = '%s' % linkedname try: path = inspect.getabsfile(object) url = urllib.parse.quote(path) @@ -680,9 +870,7 @@ def docmodule(self, object, name=None, mod=None, *ignored): docloc = '
Module Reference' % locals() else: docloc = '' - result = self.heading( - head, '#ffffff', '#7799ee', - 'index
' + filelink + docloc) + result = self.heading(head, 'index
' + filelink + docloc) modules = inspect.getmembers(object, inspect.ismodule) @@ -704,9 +892,10 @@ def docmodule(self, object, name=None, mod=None, *ignored): cdict[key] = cdict[base] = modname + '.html#' + key funcs, fdict = [], {} for key, value in inspect.getmembers(object, inspect.isroutine): - # if __all__ exists, believe it. Otherwise use old heuristic. - if (all is not None or - inspect.isbuiltin(value) or inspect.getmodule(value) is object): + # if __all__ exists, believe it. Otherwise use a heuristic. + if (all is not None + or inspect.isbuiltin(value) + or (inspect.getmodule(value) or object) is object): if visiblename(key, all, object): funcs.append((key, value)) fdict[key] = '#-' + key @@ -717,7 +906,7 @@ def docmodule(self, object, name=None, mod=None, *ignored): data.append((key, value)) doc = self.markup(getdoc(object), self.preformat, fdict, cdict) - doc = doc and '%s' % doc + doc = doc and '%s' % doc result = result + '

%s

\n' % doc if hasattr(object, '__path__'): @@ -727,12 +916,12 @@ def docmodule(self, object, name=None, mod=None, *ignored): modpkgs.sort() contents = self.multicolumn(modpkgs, self.modpkglink) result = result + self.bigsection( - 'Package Contents', '#ffffff', '#aa55cc', contents) + 'Package Contents', 'pkg-content', contents) elif modules: contents = self.multicolumn( modules, lambda t: self.modulelink(t[1])) result = result + self.bigsection( - 'Modules', '#ffffff', '#aa55cc', contents) + 'Modules', 'pkg-content', contents) if classes: classlist = [value for (key, value) in classes] @@ -741,27 +930,25 @@ def docmodule(self, object, name=None, mod=None, *ignored): for key, value in classes: contents.append(self.document(value, key, name, fdict, cdict)) result = result + self.bigsection( - 'Classes', '#ffffff', '#ee77aa', ' '.join(contents)) + 'Classes', 'index', ' '.join(contents)) if funcs: contents = [] for key, value in funcs: contents.append(self.document(value, key, name, fdict, cdict)) result = result + self.bigsection( - 'Functions', '#ffffff', '#eeaa77', ' '.join(contents)) + 'Functions', 'functions', ' '.join(contents)) if data: contents = [] for key, value in data: contents.append(self.document(value, key)) result = result + self.bigsection( - 'Data', '#ffffff', '#55aa55', '
\n'.join(contents)) + 'Data', 'data', '
\n'.join(contents)) if hasattr(object, '__author__'): contents = self.markup(str(object.__author__), self.preformat) - result = result + self.bigsection( - 'Author', '#ffffff', '#7799ee', contents) + result = result + self.bigsection('Author', 'author', contents) if hasattr(object, '__credits__'): contents = self.markup(str(object.__credits__), self.preformat) - result = result + self.bigsection( - 'Credits', '#ffffff', '#7799ee', contents) + result = result + self.bigsection('Credits', 'credits', contents) return result @@ -806,10 +993,10 @@ def spill(msg, attrs, predicate): except Exception: # Some descriptors may meet a failure in their __get__. # (bug #1785) - push(self._docdescriptor(name, value, mod)) + push(self.docdata(value, name, mod)) else: push(self.document(value, name, mod, - funcs, classes, mdict, object)) + funcs, classes, mdict, object, homecls)) push('\n') return attrs @@ -819,7 +1006,7 @@ def spilldescriptors(msg, attrs, predicate): hr.maybe() push(msg) for name, kind, homecls, value in ok: - push(self._docdescriptor(name, value, mod)) + push(self.docdata(value, name, mod)) return attrs def spilldata(msg, attrs, predicate): @@ -829,16 +1016,13 @@ def spilldata(msg, attrs, predicate): push(msg) for name, kind, homecls, value in ok: base = self.docother(getattr(object, name), name, mod) - if callable(value) or inspect.isdatadescriptor(value): - doc = getattr(value, "__doc__", None) - else: - doc = None - if doc is None: + doc = getdoc(value) + if not doc: push('
%s
\n' % base) else: doc = self.markup(getdoc(value), self.preformat, funcs, classes, mdict) - doc = '
%s' % doc + doc = '
%s' % doc push('
%s%s
\n' % (base, doc)) push('\n') return attrs @@ -870,7 +1054,7 @@ def spilldata(msg, attrs, predicate): thisclass = attrs[0][2] attrs, inherited = _split_list(attrs, lambda t: t[2] is thisclass) - if thisclass is builtins.object: + if object is not builtins.object and thisclass is builtins.object: attrs = inherited continue elif thisclass is object: @@ -889,6 +1073,8 @@ def spilldata(msg, attrs, predicate): lambda t: t[1] == 'class method') attrs = spill('Static methods %s' % tag, attrs, lambda t: t[1] == 'static method') + attrs = spilldescriptors("Readonly properties %s" % tag, attrs, + lambda t: t[1] == 'readonly property') attrs = spilldescriptors('Data descriptors %s' % tag, attrs, lambda t: t[1] == 'data descriptor') attrs = spilldata('Data and other attributes %s' % tag, attrs, @@ -909,100 +1095,129 @@ def spilldata(msg, attrs, predicate): for base in bases: parents.append(self.classlink(base, object.__module__)) title = title + '(%s)' % ', '.join(parents) - doc = self.markup(getdoc(object), self.preformat, funcs, classes, mdict) - doc = doc and '%s
 
' % doc - return self.section(title, '#000000', '#ffc8d8', contents, 3, doc) + decl = '' + argspec = _getargspec(object) + if argspec and argspec != '()': + decl = name + self.escape(argspec) + '\n\n' + + doc = getdoc(object) + if decl: + doc = decl + (doc or '') + doc = self.markup(doc, self.preformat, funcs, classes, mdict) + doc = doc and '%s
 
' % doc + + return self.section(title, 'title', contents, 3, doc) def formatvalue(self, object): """Format an argument default value as text.""" return self.grey('=' + self.repr(object)) def docroutine(self, object, name=None, mod=None, - funcs={}, classes={}, methods={}, cl=None): + funcs={}, classes={}, methods={}, cl=None, homecls=None): """Produce HTML documentation for a function or method object.""" realname = object.__name__ name = name or realname - anchor = (cl and cl.__name__ or '') + '-' + name + if homecls is None: + homecls = cl + anchor = ('' if cl is None else cl.__name__) + '-' + name note = '' - skipdocs = 0 + skipdocs = False + imfunc = None if _is_bound_method(object): - imclass = object.__self__.__class__ - if cl: - if imclass is not cl: - note = ' from ' + self.classlink(imclass, mod) + imself = object.__self__ + if imself is cl: + imfunc = getattr(object, '__func__', None) + elif inspect.isclass(imself): + note = ' class method of %s' % self.classlink(imself, mod) else: - if object.__self__ is not None: - note = ' method of %s instance' % self.classlink( - object.__self__.__class__, mod) - else: - note = ' unbound %s method' % self.classlink(imclass,mod) + note = ' method of %s instance' % self.classlink( + imself.__class__, mod) + elif (inspect.ismethoddescriptor(object) or + inspect.ismethodwrapper(object)): + try: + objclass = object.__objclass__ + except AttributeError: + pass + else: + if cl is None: + note = ' unbound %s method' % self.classlink(objclass, mod) + elif objclass is not homecls: + note = ' from ' + self.classlink(objclass, mod) + else: + imfunc = object + if inspect.isfunction(imfunc) and homecls is not None and ( + imfunc.__module__ != homecls.__module__ or + imfunc.__qualname__ != homecls.__qualname__ + '.' + realname): + pname = self.parentlink(imfunc, mod) + if pname: + note = ' from %s' % pname + + if (inspect.iscoroutinefunction(object) or + inspect.isasyncgenfunction(object)): + asyncqualifier = 'async ' + else: + asyncqualifier = '' if name == realname: title = '%s' % (anchor, realname) else: - if cl and inspect.getattr_static(cl, realname, []) is object: + if (cl is not None and + inspect.getattr_static(cl, realname, []) is object): reallink = '%s' % ( cl.__name__ + '-' + realname, realname) - skipdocs = 1 + skipdocs = True + if note.startswith(' from '): + note = '' else: reallink = realname title = '%s = %s' % ( anchor, name, reallink) argspec = None if inspect.isroutine(object): - try: - signature = inspect.signature(object) - except (ValueError, TypeError): - signature = None - if signature: - argspec = str(signature) - if realname == '': - title = '%s lambda ' % name - # XXX lambda's won't usually have func_annotations['return'] - # since the syntax doesn't support but it is possible. - # So removing parentheses isn't truly safe. + argspec = _getargspec(object) + if argspec and realname == '': + title = '%s lambda ' % name + # XXX lambda's won't usually have func_annotations['return'] + # since the syntax doesn't support but it is possible. + # So removing parentheses isn't truly safe. + if not object.__annotations__: argspec = argspec[1:-1] # remove parentheses if not argspec: argspec = '(...)' - decl = title + self.escape(argspec) + (note and self.grey( - '%s' % note)) + decl = asyncqualifier + title + self.escape(argspec) + (note and + self.grey('%s' % note)) if skipdocs: return '
%s
\n' % decl else: doc = self.markup( getdoc(object), self.preformat, funcs, classes, methods) - doc = doc and '
%s
' % doc + doc = doc and '
%s
' % doc return '
%s
%s
\n' % (decl, doc) - def _docdescriptor(self, name, value, mod): + def docdata(self, object, name=None, mod=None, cl=None, *ignored): + """Produce html documentation for a data descriptor.""" results = [] push = results.append if name: push('
%s
\n' % name) - if value.__doc__ is not None: - doc = self.markup(getdoc(value), self.preformat) - push('
%s
\n' % doc) + doc = self.markup(getdoc(object), self.preformat) + if doc: + push('
%s
\n' % doc) push('
\n') return ''.join(results) - def docproperty(self, object, name=None, mod=None, cl=None): - """Produce html documentation for a property.""" - return self._docdescriptor(name, object, mod) + docproperty = docdata def docother(self, object, name=None, mod=None, *ignored): """Produce HTML documentation for a data object.""" lhs = name and '%s = ' % name or '' return lhs + self.repr(object) - def docdata(self, object, name=None, mod=None, cl=None): - """Produce html documentation for a data descriptor.""" - return self._docdescriptor(name, object, mod) - def index(self, dir, shadowed=None): """Generate an HTML index for a directory of modules.""" modpkgs = [] @@ -1016,7 +1231,7 @@ def index(self, dir, shadowed=None): modpkgs.sort() contents = self.multicolumn(modpkgs, self.modpkglink) - return self.bigsection(dir, '#ffffff', '#ee77aa', contents) + return self.bigsection(dir, 'index', contents) # -------------------------------------------- text documentation generator @@ -1067,8 +1282,7 @@ def bold(self, text): def indent(self, text, prefix=' '): """Indent text by prepending a given prefix to each line.""" if not text: return '' - lines = [prefix + line for line in text.split('\n')] - if lines: lines[-1] = lines[-1].rstrip() + lines = [(prefix + line).rstrip() for line in text.split('\n')] return '\n'.join(lines) def section(self, title, contents): @@ -1082,19 +1296,19 @@ def formattree(self, tree, modname, parent=None, prefix=''): """Render in text a class tree as returned by inspect.getclasstree().""" result = '' for entry in tree: - if type(entry) is type(()): + if isinstance(entry, tuple): c, bases = entry result = result + prefix + classname(c, modname) if bases and bases != (parent,): parents = (classname(c, modname) for c in bases) result = result + '(%s)' % ', '.join(parents) result = result + '\n' - elif type(entry) is type([]): + elif isinstance(entry, list): result = result + self.formattree( entry, modname, c, prefix + ' ') return result - def docmodule(self, object, name=None, mod=None): + def docmodule(self, object, name=None, mod=None, *ignored): """Produce text documentation for a given module object.""" name = object.__name__ # ignore the passed-in name synop, desc = splitdoc(getdoc(object)) @@ -1123,9 +1337,10 @@ def docmodule(self, object, name=None, mod=None): classes.append((key, value)) funcs = [] for key, value in inspect.getmembers(object, inspect.isroutine): - # if __all__ exists, believe it. Otherwise use old heuristic. - if (all is not None or - inspect.isbuiltin(value) or inspect.getmodule(value) is object): + # if __all__ exists, believe it. Otherwise use a heuristic. + if (all is not None + or inspect.isbuiltin(value) + or (inspect.getmodule(value) or object) is object): if visiblename(key, all, object): funcs.append((key, value)) data = [] @@ -1212,10 +1427,17 @@ def makename(c, m=object.__module__): parents = map(makename, bases) title = title + '(%s)' % ', '.join(parents) - doc = getdoc(object) - contents = doc and [doc + '\n'] or [] + contents = [] push = contents.append + argspec = _getargspec(object) + if argspec and argspec != '()': + push(name + argspec + '\n') + + doc = getdoc(object) + if doc: + push(doc + '\n') + # List the mro, if non-trivial. mro = deque(inspect.getmro(object)) if len(mro) > 2: @@ -1224,6 +1446,25 @@ def makename(c, m=object.__module__): push(' ' + makename(base)) push('') + # List the built-in subclasses, if any: + subclasses = sorted( + (str(cls.__name__) for cls in type.__subclasses__(object) + if (not cls.__name__.startswith("_") and + getattr(cls, '__module__', '') == "builtins")), + key=str.lower + ) + no_of_subclasses = len(subclasses) + MAX_SUBCLASSES_TO_DISPLAY = 4 + if subclasses: + push("Built-in subclasses:") + for subclassname in subclasses[:MAX_SUBCLASSES_TO_DISPLAY]: + push(' ' + subclassname) + if no_of_subclasses > MAX_SUBCLASSES_TO_DISPLAY: + push(' ... and ' + + str(no_of_subclasses - MAX_SUBCLASSES_TO_DISPLAY) + + ' other subclasses') + push('') + # Cute little class to pump out a horizontal rule between sections. class HorizontalRule: def __init__(self): @@ -1245,10 +1486,10 @@ def spill(msg, attrs, predicate): except Exception: # Some descriptors may meet a failure in their __get__. # (bug #1785) - push(self._docdescriptor(name, value, mod)) + push(self.docdata(value, name, mod)) else: push(self.document(value, - name, mod, object)) + name, mod, object, homecls)) return attrs def spilldescriptors(msg, attrs, predicate): @@ -1257,7 +1498,7 @@ def spilldescriptors(msg, attrs, predicate): hr.maybe() push(msg) for name, kind, homecls, value in ok: - push(self._docdescriptor(name, value, mod)) + push(self.docdata(value, name, mod)) return attrs def spilldata(msg, attrs, predicate): @@ -1266,10 +1507,7 @@ def spilldata(msg, attrs, predicate): hr.maybe() push(msg) for name, kind, homecls, value in ok: - if callable(value) or inspect.isdatadescriptor(value): - doc = getdoc(value) - else: - doc = None + doc = getdoc(value) try: obj = getattr(object, name) except AttributeError: @@ -1289,7 +1527,7 @@ def spilldata(msg, attrs, predicate): thisclass = attrs[0][2] attrs, inherited = _split_list(attrs, lambda t: t[2] is thisclass) - if thisclass is builtins.object: + if object is not builtins.object and thisclass is builtins.object: attrs = inherited continue elif thisclass is object: @@ -1307,6 +1545,8 @@ def spilldata(msg, attrs, predicate): lambda t: t[1] == 'class method') attrs = spill("Static methods %s:\n" % tag, attrs, lambda t: t[1] == 'static method') + attrs = spilldescriptors("Readonly properties %s:\n" % tag, attrs, + lambda t: t[1] == 'readonly property') attrs = spilldescriptors("Data descriptors %s:\n" % tag, attrs, lambda t: t[1] == 'data descriptor') attrs = spilldata("Data and other attributes %s:\n" % tag, attrs, @@ -1324,48 +1564,73 @@ def formatvalue(self, object): """Format an argument default value as text.""" return '=' + self.repr(object) - def docroutine(self, object, name=None, mod=None, cl=None): + def docroutine(self, object, name=None, mod=None, cl=None, homecls=None): """Produce text documentation for a function or method object.""" realname = object.__name__ name = name or realname + if homecls is None: + homecls = cl note = '' - skipdocs = 0 + skipdocs = False + imfunc = None if _is_bound_method(object): - imclass = object.__self__.__class__ - if cl: - if imclass is not cl: - note = ' from ' + classname(imclass, mod) + imself = object.__self__ + if imself is cl: + imfunc = getattr(object, '__func__', None) + elif inspect.isclass(imself): + note = ' class method of %s' % classname(imself, mod) else: - if object.__self__ is not None: - note = ' method of %s instance' % classname( - object.__self__.__class__, mod) - else: - note = ' unbound %s method' % classname(imclass,mod) + note = ' method of %s instance' % classname( + imself.__class__, mod) + elif (inspect.ismethoddescriptor(object) or + inspect.ismethodwrapper(object)): + try: + objclass = object.__objclass__ + except AttributeError: + pass + else: + if cl is None: + note = ' unbound %s method' % classname(objclass, mod) + elif objclass is not homecls: + note = ' from ' + classname(objclass, mod) + else: + imfunc = object + if inspect.isfunction(imfunc) and homecls is not None and ( + imfunc.__module__ != homecls.__module__ or + imfunc.__qualname__ != homecls.__qualname__ + '.' + realname): + pname = parentname(imfunc, mod) + if pname: + note = ' from %s' % pname + + if (inspect.iscoroutinefunction(object) or + inspect.isasyncgenfunction(object)): + asyncqualifier = 'async ' + else: + asyncqualifier = '' if name == realname: title = self.bold(realname) else: - if cl and inspect.getattr_static(cl, realname, []) is object: - skipdocs = 1 + if (cl is not None and + inspect.getattr_static(cl, realname, []) is object): + skipdocs = True + if note.startswith(' from '): + note = '' title = self.bold(name) + ' = ' + realname argspec = None if inspect.isroutine(object): - try: - signature = inspect.signature(object) - except (ValueError, TypeError): - signature = None - if signature: - argspec = str(signature) - if realname == '': - title = self.bold(name) + ' lambda ' - # XXX lambda's won't usually have func_annotations['return'] - # since the syntax doesn't support but it is possible. - # So removing parentheses isn't truly safe. - argspec = argspec[1:-1] # remove parentheses + argspec = _getargspec(object) + if argspec and realname == '': + title = self.bold(name) + ' lambda ' + # XXX lambda's won't usually have func_annotations['return'] + # since the syntax doesn't support but it is possible. + # So removing parentheses isn't truly safe. + if not object.__annotations__: + argspec = argspec[1:-1] if not argspec: argspec = '(...)' - decl = title + argspec + note + decl = asyncqualifier + title + argspec + note if skipdocs: return decl + '\n' @@ -1373,28 +1638,24 @@ def docroutine(self, object, name=None, mod=None, cl=None): doc = getdoc(object) or '' return decl + '\n' + (doc and self.indent(doc).rstrip() + '\n') - def _docdescriptor(self, name, value, mod): + def docdata(self, object, name=None, mod=None, cl=None, *ignored): + """Produce text documentation for a data descriptor.""" results = [] push = results.append if name: push(self.bold(name)) push('\n') - doc = getdoc(value) or '' + doc = getdoc(object) or '' if doc: push(self.indent(doc)) push('\n') return ''.join(results) - def docproperty(self, object, name=None, mod=None, cl=None): - """Produce text documentation for a property.""" - return self._docdescriptor(name, object, mod) + docproperty = docdata - def docdata(self, object, name=None, mod=None, cl=None): - """Produce text documentation for a data descriptor.""" - return self._docdescriptor(name, object, mod) - - def docother(self, object, name=None, mod=None, parent=None, maxlen=None, doc=None): + def docother(self, object, name=None, mod=None, parent=None, *ignored, + maxlen=None, doc=None): """Produce text documentation for a data object.""" repr = self.repr(object) if maxlen: @@ -1402,8 +1663,10 @@ def docother(self, object, name=None, mod=None, parent=None, maxlen=None, doc=No chop = maxlen - len(line) if chop < 0: repr = repr[:chop] + '...' line = (name and self.bold(name) + ' = ' or '') + repr - if doc is not None: - line += '\n' + self.indent(str(doc)) + if not doc: + doc = getdoc(object) + if doc: + line += '\n' + self.indent(str(doc)) + '\n' return line class _PlainTextDoc(TextDoc): @@ -1413,136 +1676,11 @@ def bold(self, text): # --------------------------------------------------------- user interfaces -def pager(text): +def pager(text, title=''): """The first time this is called, determine what kind of pager to use.""" global pager - pager = getpager() - pager(text) - -def getpager(): - """Decide what method to use for paging through text.""" - if not hasattr(sys.stdin, "isatty"): - return plainpager - if not hasattr(sys.stdout, "isatty"): - return plainpager - if not sys.stdin.isatty() or not sys.stdout.isatty(): - return plainpager - use_pager = os.environ.get('MANPAGER') or os.environ.get('PAGER') - if use_pager: - if sys.platform == 'win32': # pipes completely broken in Windows - return lambda text: tempfilepager(plain(text), use_pager) - elif os.environ.get('TERM') in ('dumb', 'emacs'): - return lambda text: pipepager(plain(text), use_pager) - else: - return lambda text: pipepager(text, use_pager) - if os.environ.get('TERM') in ('dumb', 'emacs'): - return plainpager - if sys.platform == 'win32': - return lambda text: tempfilepager(plain(text), 'more <') - if hasattr(os, 'system') and os.system('(less) 2>/dev/null') == 0: - return lambda text: pipepager(text, 'less') - - import tempfile - (fd, filename) = tempfile.mkstemp() - os.close(fd) - try: - if hasattr(os, 'system') and os.system('more "%s"' % filename) == 0: - return lambda text: pipepager(text, 'more') - else: - return ttypager - finally: - os.unlink(filename) - -def plain(text): - """Remove boldface formatting from text.""" - return re.sub('.\b', '', text) - -def pipepager(text, cmd): - """Page through text by feeding it to another program.""" - import subprocess - proc = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE) - try: - with io.TextIOWrapper(proc.stdin, errors='backslashreplace') as pipe: - try: - pipe.write(text) - except KeyboardInterrupt: - # We've hereby abandoned whatever text hasn't been written, - # but the pager is still in control of the terminal. - pass - except OSError: - pass # Ignore broken pipes caused by quitting the pager program. - while True: - try: - proc.wait() - break - except KeyboardInterrupt: - # Ignore ctl-c like the pager itself does. Otherwise the pager is - # left running and the terminal is in raw mode and unusable. - pass - -def tempfilepager(text, cmd): - """Page through text by invoking a program on a temporary file.""" - import tempfile - filename = tempfile.mktemp() - with open(filename, 'w', errors='backslashreplace') as file: - file.write(text) - try: - os.system(cmd + ' "' + filename + '"') - finally: - os.unlink(filename) - -def _escape_stdout(text): - # Escape non-encodable characters to avoid encoding errors later - encoding = getattr(sys.stdout, 'encoding', None) or 'utf-8' - return text.encode(encoding, 'backslashreplace').decode(encoding) - -def ttypager(text): - """Page through text on a text terminal.""" - lines = plain(_escape_stdout(text)).split('\n') - try: - import tty - fd = sys.stdin.fileno() - old = tty.tcgetattr(fd) - tty.setcbreak(fd) - getchar = lambda: sys.stdin.read(1) - except (ImportError, AttributeError, io.UnsupportedOperation): - tty = None - getchar = lambda: sys.stdin.readline()[:-1][:1] - - try: - try: - h = int(os.environ.get('LINES', 0)) - except ValueError: - h = 0 - if h <= 1: - h = 25 - r = inc = h - 1 - sys.stdout.write('\n'.join(lines[:inc]) + '\n') - while lines[r:]: - sys.stdout.write('-- more --') - sys.stdout.flush() - c = getchar() - - if c in ('q', 'Q'): - sys.stdout.write('\r \r') - break - elif c in ('\r', '\n'): - sys.stdout.write('\r \r' + lines[r] + '\n') - r = r + 1 - continue - if c in ('b', 'B', '\x1b'): - r = r - inc - inc - if r < 0: r = 0 - sys.stdout.write('\n' + '\n'.join(lines[r:r+inc]) + '\n') - r = r + inc - - finally: - if tty: - tty.tcsetattr(fd, tty.TCSAFLUSH, old) - -def plainpager(text): - """Simply print unformatted text. This is the ultimate fallback.""" - sys.stdout.write(plain(_escape_stdout(text))) + pager = get_pager() + pager(text, title) def describe(thing): """Produce a short description of the given thing.""" @@ -1569,6 +1707,13 @@ def describe(thing): return 'function ' + thing.__name__ if inspect.ismethod(thing): return 'method ' + thing.__name__ + if inspect.ismethodwrapper(thing): + return 'method wrapper ' + thing.__name__ + if inspect.ismethoddescriptor(thing): + try: + return 'method descriptor ' + thing.__name__ + except AttributeError: + pass return type(thing).__name__ def locate(path, forceload=0): @@ -1626,36 +1771,49 @@ def render_doc(thing, title='Python Library Documentation: %s', forceload=0, if not (inspect.ismodule(object) or inspect.isclass(object) or inspect.isroutine(object) or - inspect.isgetsetdescriptor(object) or - inspect.ismemberdescriptor(object) or - isinstance(object, property)): + inspect.isdatadescriptor(object) or + _getdoc(object)): # If the passed object is a piece of data or an instance, # document its available methods instead of its value. - object = type(object) - desc += ' object' + if hasattr(object, '__origin__'): + object = object.__origin__ + else: + object = type(object) + desc += ' object' return title % desc + '\n\n' + renderer.document(object, name) def doc(thing, title='Python Library Documentation: %s', forceload=0, - output=None): + output=None, is_cli=False): """Display text documentation, given an object or a path to an object.""" - try: - if output is None: - pager(render_doc(thing, title, forceload)) - else: - output.write(render_doc(thing, title, forceload, plaintext)) - except (ImportError, ErrorDuringImport) as value: - print(value) + if output is None: + try: + if isinstance(thing, str): + what = thing + else: + what = getattr(thing, '__qualname__', None) + if not isinstance(what, str): + what = getattr(thing, '__name__', None) + if not isinstance(what, str): + what = type(thing).__name__ + ' object' + pager(render_doc(thing, title, forceload), f'Help on {what!s}') + except ImportError as exc: + if is_cli: + raise + print(exc) + else: + try: + s = render_doc(thing, title, forceload, plaintext) + except ImportError as exc: + s = str(exc) + output.write(s) def writedoc(thing, forceload=0): """Write HTML documentation to a file in the current directory.""" - try: - object, name = resolve(thing, forceload) - page = html.page(describe(object), html.document(object, name)) - with open(name + '.html', 'w', encoding='utf-8') as file: - file.write(page) - print('wrote', name + '.html') - except (ImportError, ErrorDuringImport) as value: - print(value) + object, name = resolve(thing, forceload) + page = html.page(describe(object), html.document(object, name)) + with open(name + '.html', 'w', encoding='utf-8') as file: + file.write(page) + print('wrote', name + '.html') def writedocs(dir, pkgpath='', done=None): """Write out HTML documentation for all modules in a directory tree.""" @@ -1664,6 +1822,37 @@ def writedocs(dir, pkgpath='', done=None): writedoc(modname) return + +def _introdoc(): + import textwrap + ver = '%d.%d' % sys.version_info[:2] + if os.environ.get('PYTHON_BASIC_REPL'): + pyrepl_keys = '' + else: + # Additional help for keyboard shortcuts if enhanced REPL is used. + pyrepl_keys = ''' + You can use the following keyboard shortcuts at the main interpreter prompt. + F1: enter interactive help, F2: enter history browsing mode, F3: enter paste + mode (press again to exit). + ''' + return textwrap.dedent(f'''\ + Welcome to Python {ver}'s help utility! If this is your first time using + Python, you should definitely check out the tutorial at + https://docs.python.org/{ver}/tutorial/. + + Enter the name of any module, keyword, or topic to get help on writing + Python programs and using Python modules. To get a list of available + modules, keywords, symbols, or topics, enter "modules", "keywords", + "symbols", or "topics". + {pyrepl_keys} + Each module also comes with a one-line summary of what it does; to list + the modules whose name or summary contain a given string such as "spam", + enter "modules spam". + + To quit this help utility and return to the interpreter, + enter "q", "quit" or "exit". + ''') + class Helper: # These dictionaries map a topic name to either an alias, or a tuple @@ -1672,7 +1861,7 @@ class Helper: # in pydoc_data/topics.py. # # CAUTION: if you change one of these dictionaries, be sure to adapt the - # list of needed labels in Doc/tools/pyspecific.py and + # list of needed labels in Doc/tools/extensions/pyspecific.py and # regenerate the pydoc_data/topics.py file by running # make pydoc-topics # in Doc/ and copying the output file into the Lib/ directory. @@ -1684,6 +1873,8 @@ class Helper: 'and': 'BOOLEAN', 'as': 'with', 'assert': ('assert', ''), + 'async': ('async', ''), + 'await': ('await', ''), 'break': ('break', 'while for'), 'class': ('class', 'CLASSES SPECIALMETHODS'), 'continue': ('continue', 'while for'), @@ -1735,6 +1926,7 @@ class Helper: ':': 'SLICINGS DICTIONARYLITERALS', '@': 'def class', '\\': 'STRINGS', + ':=': 'ASSIGNMENTEXPRESSIONS', '_': 'PRIVATENAMES', '__': 'PRIVATENAMES SPECIALMETHODS', '`': 'BACKQUOTES', @@ -1749,6 +1941,7 @@ class Helper: if topic not in topics: topics = topics + ' ' + topic symbols[symbol] = topics + del topic, symbols_, symbol, topics topics = { 'TYPES': ('types', 'STRINGS UNICODE NUMBERS SEQUENCES MAPPINGS ' @@ -1827,6 +2020,7 @@ class Helper: 'ASSERTION': 'assert', 'ASSIGNMENT': ('assignment', 'AUGMENTEDASSIGNMENT'), 'AUGMENTEDASSIGNMENT': ('augassign', 'NUMBERMETHODS'), + 'ASSIGNMENTEXPRESSIONS': ('assignment-expressions', ''), 'DELETION': 'del', 'RETURNING': 'return', 'IMPORTING': 'import', @@ -1841,8 +2035,13 @@ def __init__(self, input=None, output=None): self._input = input self._output = output - input = property(lambda self: self._input or sys.stdin) - output = property(lambda self: self._output or sys.stdout) + @property + def input(self): + return self._input or sys.stdin + + @property + def output(self): + return self._output or sys.stdout def __repr__(self): if inspect.stack()[1][3] == '?': @@ -1854,7 +2053,10 @@ def __repr__(self): _GoInteractive = object() def __call__(self, request=_GoInteractive): if request is not self._GoInteractive: - self.help(request) + try: + self.help(request) + except ImportError as err: + self.output.write(f'{err}\n') else: self.intro() self.interact() @@ -1870,17 +2072,18 @@ def interact(self): while True: try: request = self.getline('help> ') - if not request: break except (KeyboardInterrupt, EOFError): break request = request.strip() + if not request: + continue # back to the prompt # Make sure significant trailing quoting marks of literals don't # get deleted while cleaning input if (len(request) > 2 and request[0] == request[-1] in ("'", '"') and request[0] not in request[1:-1]): request = request[1:-1] - if request.lower() in ('q', 'quit'): break + if request.lower() in ('q', 'quit', 'exit'): break if request == 'help': self.intro() else: @@ -1895,8 +2098,8 @@ def getline(self, prompt): self.output.flush() return self.input.readline() - def help(self, request): - if type(request) is type(''): + def help(self, request, is_cli=False): + if isinstance(request, str): request = request.strip() if request == 'keywords': self.listkeywords() elif request == 'symbols': self.listsymbols() @@ -1907,34 +2110,20 @@ def help(self, request): elif request in self.symbols: self.showsymbol(request) elif request in ['True', 'False', 'None']: # special case these keywords since they are objects too - doc(eval(request), 'Help on %s:') + doc(eval(request), 'Help on %s:', output=self._output, is_cli=is_cli) elif request in self.keywords: self.showtopic(request) elif request in self.topics: self.showtopic(request) - elif request: doc(request, 'Help on %s:', output=self._output) - else: doc(str, 'Help on %s:', output=self._output) + elif request: doc(request, 'Help on %s:', output=self._output, is_cli=is_cli) + else: doc(str, 'Help on %s:', output=self._output, is_cli=is_cli) elif isinstance(request, Helper): self() - else: doc(request, 'Help on %s:', output=self._output) + else: doc(request, 'Help on %s:', output=self._output, is_cli=is_cli) self.output.write('\n') def intro(self): - self.output.write(''' -Welcome to Python {0}'s help utility! - -If this is your first time using Python, you should definitely check out -the tutorial on the Internet at https://docs.python.org/{0}/tutorial/. - -Enter the name of any module, keyword, or topic to get help on writing -Python programs and using Python modules. To quit this help utility and -return to the interpreter, just type "quit". - -To get a list of available modules, keywords, symbols, or topics, type -"modules", "keywords", "symbols", or "topics". Each module also comes -with a one-line summary of what it does; to list the modules whose name -or summary contain a given string such as "spam", type "modules spam". -'''.format('%d.%d' % sys.version_info[:2])) + self.output.write(_introdoc()) def list(self, items, columns=4, width=80): - items = list(sorted(items)) + items = sorted(items) colw = width // columns rows = (len(items) + columns - 1) // columns for row in range(rows): @@ -1966,7 +2155,7 @@ def listtopics(self): Here is a list of available topics. Enter any topic name to get more help. ''') - self.list(self.topics.keys()) + self.list(self.topics.keys(), columns=3) def showtopic(self, topic, more_xrefs=''): try: @@ -1981,7 +2170,7 @@ def showtopic(self, topic, more_xrefs=''): if not target: self.output.write('no documentation found for %s\n' % repr(topic)) return - if type(target) is type(''): + if isinstance(target, str): return self.showtopic(target, more_xrefs) label, xrefs = target @@ -1998,7 +2187,11 @@ def showtopic(self, topic, more_xrefs=''): text = 'Related help topics: ' + ', '.join(xrefs.split()) + '\n' wrapped_text = textwrap.wrap(text, 72) doc += '\n%s\n' % '\n'.join(wrapped_text) - pager(doc) + + if self._output is None: + pager(doc, f'Help on {topic!s}') + else: + self.output.write(doc) def _gettopic(self, topic, more_xrefs=''): """Return unbuffered tuple of (topic, xrefs). @@ -2090,7 +2283,7 @@ def run(self, callback, key=None, completer=None, onerror=None): callback(None, modname, '') else: try: - spec = pkgutil._get_spec(importer, modname) + spec = importer.find_spec(modname) except SyntaxError: # raised by tests for bad coding cookies or BOM continue @@ -2135,13 +2328,13 @@ def onerror(modname): warnings.filterwarnings('ignore') # ignore problems during import ModuleScanner().run(callback, key, onerror=onerror) -# --------------------------------------- enhanced Web browser interface +# --------------------------------------- enhanced web browser interface -def _start_server(urlhandler, port): +def _start_server(urlhandler, hostname, port): """Start an HTTP server thread on a specific port. Start an HTML/text server thread, so HTML or text documents can be - browsed dynamically and interactively with a Web browser. Example use: + browsed dynamically and interactively with a web browser. Example use: >>> import time >>> import pydoc @@ -2177,14 +2370,14 @@ def _start_server(urlhandler, port): Let the server do its thing. We just need to monitor its status. Use time.sleep so the loop doesn't hog the CPU. - >>> starttime = time.time() + >>> starttime = time.monotonic() >>> timeout = 1 #seconds This is a short timeout for testing purposes. >>> while serverthread.serving: ... time.sleep(.01) - ... if serverthread.serving and time.time() - starttime > timeout: + ... if serverthread.serving and time.monotonic() - starttime > timeout: ... serverthread.stop() ... break @@ -2222,8 +2415,8 @@ def log_message(self, *args): class DocServer(http.server.HTTPServer): - def __init__(self, port, callback): - self.host = 'localhost' + def __init__(self, host, port, callback): + self.host = host self.address = (self.host, port) self.callback = callback self.base.__init__(self, self.address, self.handler) @@ -2243,12 +2436,14 @@ def server_activate(self): class ServerThread(threading.Thread): - def __init__(self, urlhandler, port): + def __init__(self, urlhandler, host, port): self.urlhandler = urlhandler + self.host = host self.port = int(port) threading.Thread.__init__(self) self.serving = False self.error = None + self.docserver = None def run(self): """Start the server.""" @@ -2257,11 +2452,11 @@ def run(self): DocServer.handler = DocHandler DocHandler.MessageClass = email.message.Message DocHandler.urlhandler = staticmethod(self.urlhandler) - docsvr = DocServer(self.port, self.ready) + docsvr = DocServer(self.host, self.port, self.ready) self.docserver = docsvr docsvr.serve_until_quit() - except Exception as e: - self.error = e + except Exception as err: + self.error = err def ready(self, server): self.serving = True @@ -2279,11 +2474,11 @@ def stop(self): self.serving = False self.url = None - thread = ServerThread(urlhandler, port) + thread = ServerThread(urlhandler, hostname, port) thread.start() - # Wait until thread.serving is True to make sure we are - # really up before returning. - while not thread.error and not thread.serving: + # Wait until thread.serving is True and thread.docserver is set + # to make sure we are really up before returning. + while not thread.error and not (thread.serving and thread.docserver): time.sleep(.01) return thread @@ -2306,15 +2501,14 @@ def page(self, title, contents): '' % css_path) return '''\ - -Pydoc: %s - -%s%s
%s
+ + + + +Pydoc: %s +%s%s
%s
''' % (title, css_link, html_navbar(), contents) - def filelink(self, url, path): - return '%s' % (url, path) - html = _HTMLDoc() @@ -2352,22 +2546,21 @@ def bltinlink(name): return '%s' % (name, name) heading = html.heading( - 'Index of Modules', - '#ffffff', '#7799ee') + 'Index of Modules' + ) names = [name for name in sys.builtin_module_names if name != '__main__'] contents = html.multicolumn(names, bltinlink) contents = [heading, '

' + html.bigsection( - 'Built-in Modules', '#ffffff', '#ee77aa', contents)] + 'Built-in Modules', 'index', contents)] seen = {} for dir in sys.path: contents.append(html.index(dir, seen)) contents.append( - '

pydoc by Ka-Ping Yee' - '<ping@lfw.org>') + '

pydoc by Ka-Ping Yee' + '<ping@lfw.org>

') return 'Index of Modules', ''.join(contents) def html_search(key): @@ -2392,27 +2585,14 @@ def bltinlink(name): results = [] heading = html.heading( - 'Search Results', - '#ffffff', '#7799ee') + 'Search Results', + ) for name, desc in search_result: results.append(bltinlink(name) + desc) contents = heading + html.bigsection( - 'key = %s' % key, '#ffffff', '#ee77aa', '
'.join(results)) + 'key = %s' % key, 'index', '
'.join(results)) return 'Search Results', contents - def html_getfile(path): - """Get and display a source file listing safely.""" - path = urllib.parse.unquote(path) - with tokenize.open(path) as fp: - lines = html.escape(fp.read()) - body = '
%s
' % lines - heading = html.heading( - 'File Listing', - '#ffffff', '#7799ee') - contents = heading + html.bigsection( - 'File: %s' % path, '#ffffff', '#ee77aa', body) - return 'getfile %s' % path, contents - def html_topics(): """Index of topic texts available.""" @@ -2420,20 +2600,20 @@ def bltinlink(name): return '%s' % (name, name) heading = html.heading( - 'INDEX', - '#ffffff', '#7799ee') + 'INDEX', + ) names = sorted(Helper.topics.keys()) contents = html.multicolumn(names, bltinlink) contents = heading + html.bigsection( - 'Topics', '#ffffff', '#ee77aa', contents) + 'Topics', 'index', contents) return 'Topics', contents def html_keywords(): """Index of keywords.""" heading = html.heading( - 'INDEX', - '#ffffff', '#7799ee') + 'INDEX', + ) names = sorted(Helper.keywords.keys()) def bltinlink(name): @@ -2441,7 +2621,7 @@ def bltinlink(name): contents = html.multicolumn(names, bltinlink) contents = heading + html.bigsection( - 'Keywords', '#ffffff', '#ee77aa', contents) + 'Keywords', 'index', contents) return 'Keywords', contents def html_topicpage(topic): @@ -2454,10 +2634,10 @@ def html_topicpage(topic): else: title = 'TOPIC' heading = html.heading( - '%s' % title, - '#ffffff', '#7799ee') + '%s' % title, + ) contents = '
%s
' % html.markup(contents) - contents = html.bigsection(topic , '#ffffff','#ee77aa', contents) + contents = html.bigsection(topic , 'index', contents) if xrefs: xrefs = sorted(xrefs.split()) @@ -2465,8 +2645,7 @@ def bltinlink(name): return '%s' % (name, name) xrefs = html.multicolumn(xrefs, bltinlink) - xrefs = html.section('Related help topics: ', - '#ffffff', '#ee77aa', xrefs) + xrefs = html.section('Related help topics: ', 'index', xrefs) return ('%s %s' % (title, topic), ''.join((heading, contents, xrefs))) @@ -2480,12 +2659,11 @@ def html_getobj(url): def html_error(url, exc): heading = html.heading( - 'Error', - '#ffffff', '#7799ee') + 'Error', + ) contents = '
'.join(html.escape(line) for line in format_exception_only(type(exc), exc)) - contents = heading + html.bigsection(url, '#ffffff', '#bb0000', - contents) + contents = heading + html.bigsection(url, 'error', contents) return "Error - %s" % url, contents def get_html_page(url): @@ -2504,8 +2682,6 @@ def get_html_page(url): op, _, url = url.partition('=') if op == "search?key": title, content = html_search(url) - elif op == "getfile?key": - title, content = html_getfile(url) elif op == "topic?key": # try topics first, then objects. try: @@ -2543,14 +2719,14 @@ def get_html_page(url): raise TypeError('unknown content type %r for url %s' % (content_type, url)) -def browse(port=0, *, open_browser=True): - """Start the enhanced pydoc Web server and open a Web browser. +def browse(port=0, *, open_browser=True, hostname='localhost'): + """Start the enhanced pydoc web server and open a web browser. Use port '0' to start the server on an arbitrary port. Set open_browser to False to suppress opening a browser. """ import webbrowser - serverthread = _start_server(_url_handler, port) + serverthread = _start_server(_url_handler, hostname, port) if serverthread.error: print(serverthread.error) return @@ -2583,25 +2759,58 @@ def browse(port=0, *, open_browser=True): def ispath(x): return isinstance(x, str) and x.find(os.sep) >= 0 +def _get_revised_path(given_path, argv0): + """Ensures current directory is on returned path, and argv0 directory is not + + Exception: argv0 dir is left alone if it's also pydoc's directory. + + Returns a new path entry list, or None if no adjustment is needed. + """ + # Scripts may get the current directory in their path by default if they're + # run with the -m switch, or directly from the current directory. + # The interactive prompt also allows imports from the current directory. + + # Accordingly, if the current directory is already present, don't make + # any changes to the given_path + if '' in given_path or os.curdir in given_path or os.getcwd() in given_path: + return None + + # Otherwise, add the current directory to the given path, and remove the + # script directory (as long as the latter isn't also pydoc's directory. + stdlib_dir = os.path.dirname(__file__) + script_dir = os.path.dirname(argv0) + revised_path = given_path.copy() + if script_dir in given_path and not os.path.samefile(script_dir, stdlib_dir): + revised_path.remove(script_dir) + revised_path.insert(0, os.getcwd()) + return revised_path + + +# Note: the tests only cover _get_revised_path, not _adjust_cli_path itself +def _adjust_cli_sys_path(): + """Ensures current directory is on sys.path, and __main__ directory is not. + + Exception: __main__ dir is left alone if it's also pydoc's directory. + """ + revised_path = _get_revised_path(sys.path, sys.argv[0]) + if revised_path is not None: + sys.path[:] = revised_path + + def cli(): """Command-line interface (looks at sys.argv to decide what to do).""" import getopt class BadUsage(Exception): pass - # Scripts don't get the current directory in their path by default - # unless they are run with the '-m' switch - if '' not in sys.path: - scriptdir = os.path.dirname(sys.argv[0]) - if scriptdir in sys.path: - sys.path.remove(scriptdir) - sys.path.insert(0, '.') + _adjust_cli_sys_path() try: - opts, args = getopt.getopt(sys.argv[1:], 'bk:p:w') + opts, args = getopt.getopt(sys.argv[1:], 'bk:n:p:w') writing = False start_server = False open_browser = False - port = None + port = 0 + hostname = 'localhost' for opt, val in opts: if opt == '-b': start_server = True @@ -2614,18 +2823,19 @@ class BadUsage(Exception): pass port = val if opt == '-w': writing = True + if opt == '-n': + start_server = True + hostname = val if start_server: - if port is None: - port = 0 - browse(port, open_browser=open_browser) + browse(port, hostname=hostname, open_browser=open_browser) return if not args: raise BadUsage for arg in args: if ispath(arg) and not os.path.exists(arg): print('file %r does not exist' % arg) - break + sys.exit(1) try: if ispath(arg) and os.path.isfile(arg): arg = importfile(arg) @@ -2635,9 +2845,10 @@ class BadUsage(Exception): pass else: writedoc(arg) else: - help.help(arg) - except ErrorDuringImport as value: + help.help(arg, is_cli=True) + except (ImportError, ErrorDuringImport) as value: print(value) + sys.exit(1) except (getopt.error, BadUsage): cmd = os.path.splitext(os.path.basename(sys.argv[0]))[0] @@ -2654,14 +2865,17 @@ class BadUsage(Exception): pass {cmd} -k Search for a keyword in the synopsis lines of all available modules. +{cmd} -n + Start an HTTP server with the given hostname (default: localhost). + {cmd} -p Start an HTTP server on the given port on the local machine. Port number 0 can be used to get an arbitrary unused port. {cmd} -b - Start an HTTP server on an arbitrary unused port and open a Web browser - to interactively browse documentation. The -p option can be used with - the -b option to explicitly specify the server port. + Start an HTTP server on an arbitrary unused port and open a web browser + to interactively browse documentation. This option can be used in + combination with -n and/or -p. {cmd} -w ... Write out the HTML documentation for a module to a file in the current diff --git a/Lib/pydoc_data/module_docs.py b/Lib/pydoc_data/module_docs.py new file mode 100644 index 00000000000..2a6ede3aa14 --- /dev/null +++ b/Lib/pydoc_data/module_docs.py @@ -0,0 +1,320 @@ +# Autogenerated by Sphinx on Tue Feb 3 17:32:13 2026 +# as part of the release process. + +module_docs = { + '__future__': '__future__#module-__future__', + '__main__': '__main__#module-__main__', + '_thread': '_thread#module-_thread', + '_tkinter': 'tkinter#module-_tkinter', + 'abc': 'abc#module-abc', + 'aifc': 'aifc#module-aifc', + 'annotationlib': 'annotationlib#module-annotationlib', + 'argparse': 'argparse#module-argparse', + 'array': 'array#module-array', + 'ast': 'ast#module-ast', + 'asynchat': 'asynchat#module-asynchat', + 'asyncio': 'asyncio#module-asyncio', + 'asyncore': 'asyncore#module-asyncore', + 'atexit': 'atexit#module-atexit', + 'audioop': 'audioop#module-audioop', + 'base64': 'base64#module-base64', + 'bdb': 'bdb#module-bdb', + 'binascii': 'binascii#module-binascii', + 'bisect': 'bisect#module-bisect', + 'builtins': 'builtins#module-builtins', + 'bz2': 'bz2#module-bz2', + 'cProfile': 'profile#module-cProfile', + 'calendar': 'calendar#module-calendar', + 'cgi': 'cgi#module-cgi', + 'cgitb': 'cgitb#module-cgitb', + 'chunk': 'chunk#module-chunk', + 'cmath': 'cmath#module-cmath', + 'cmd': 'cmd#module-cmd', + 'code': 'code#module-code', + 'codecs': 'codecs#module-codecs', + 'codeop': 'codeop#module-codeop', + 'collections': 'collections#module-collections', + 'collections.abc': 'collections.abc#module-collections.abc', + 'colorsys': 'colorsys#module-colorsys', + 'compileall': 'compileall#module-compileall', + 'compression': 'compression#module-compression', + 'compression.zstd': 'compression.zstd#module-compression.zstd', + 'concurrent.futures': 'concurrent.futures#module-concurrent.futures', + 'concurrent.interpreters': 'concurrent.interpreters#module-concurrent.interpreters', + 'configparser': 'configparser#module-configparser', + 'contextlib': 'contextlib#module-contextlib', + 'contextvars': 'contextvars#module-contextvars', + 'copy': 'copy#module-copy', + 'copyreg': 'copyreg#module-copyreg', + 'crypt': 'crypt#module-crypt', + 'csv': 'csv#module-csv', + 'ctypes': 'ctypes#module-ctypes', + 'curses': 'curses#module-curses', + 'curses.ascii': 'curses.ascii#module-curses.ascii', + 'curses.panel': 'curses.panel#module-curses.panel', + 'curses.textpad': 'curses#module-curses.textpad', + 'dataclasses': 'dataclasses#module-dataclasses', + 'datetime': 'datetime#module-datetime', + 'dbm': 'dbm#module-dbm', + 'dbm.dumb': 'dbm#module-dbm.dumb', + 'dbm.gnu': 'dbm#module-dbm.gnu', + 'dbm.ndbm': 'dbm#module-dbm.ndbm', + 'dbm.sqlite3': 'dbm#module-dbm.sqlite3', + 'decimal': 'decimal#module-decimal', + 'difflib': 'difflib#module-difflib', + 'dis': 'dis#module-dis', + 'distutils': 'distutils#module-distutils', + 'doctest': 'doctest#module-doctest', + 'email': 'email#module-email', + 'email.charset': 'email.charset#module-email.charset', + 'email.contentmanager': 'email.contentmanager#module-email.contentmanager', + 'email.encoders': 'email.encoders#module-email.encoders', + 'email.errors': 'email.errors#module-email.errors', + 'email.generator': 'email.generator#module-email.generator', + 'email.header': 'email.header#module-email.header', + 'email.headerregistry': 'email.headerregistry#module-email.headerregistry', + 'email.iterators': 'email.iterators#module-email.iterators', + 'email.message': 'email.message#module-email.message', + 'email.mime': 'email.mime#module-email.mime', + 'email.mime.application': 'email.mime#module-email.mime.application', + 'email.mime.audio': 'email.mime#module-email.mime.audio', + 'email.mime.base': 'email.mime#module-email.mime.base', + 'email.mime.image': 'email.mime#module-email.mime.image', + 'email.mime.message': 'email.mime#module-email.mime.message', + 'email.mime.multipart': 'email.mime#module-email.mime.multipart', + 'email.mime.nonmultipart': 'email.mime#module-email.mime.nonmultipart', + 'email.mime.text': 'email.mime#module-email.mime.text', + 'email.parser': 'email.parser#module-email.parser', + 'email.policy': 'email.policy#module-email.policy', + 'email.utils': 'email.utils#module-email.utils', + 'encodings': 'codecs#module-encodings', + 'encodings.idna': 'codecs#module-encodings.idna', + 'encodings.mbcs': 'codecs#module-encodings.mbcs', + 'encodings.utf_8_sig': 'codecs#module-encodings.utf_8_sig', + 'ensurepip': 'ensurepip#module-ensurepip', + 'enum': 'enum#module-enum', + 'errno': 'errno#module-errno', + 'faulthandler': 'faulthandler#module-faulthandler', + 'fcntl': 'fcntl#module-fcntl', + 'filecmp': 'filecmp#module-filecmp', + 'fileinput': 'fileinput#module-fileinput', + 'fnmatch': 'fnmatch#module-fnmatch', + 'fractions': 'fractions#module-fractions', + 'ftplib': 'ftplib#module-ftplib', + 'functools': 'functools#module-functools', + 'gc': 'gc#module-gc', + 'getopt': 'getopt#module-getopt', + 'getpass': 'getpass#module-getpass', + 'gettext': 'gettext#module-gettext', + 'glob': 'glob#module-glob', + 'graphlib': 'graphlib#module-graphlib', + 'grp': 'grp#module-grp', + 'gzip': 'gzip#module-gzip', + 'hashlib': 'hashlib#module-hashlib', + 'heapq': 'heapq#module-heapq', + 'hmac': 'hmac#module-hmac', + 'html': 'html#module-html', + 'html.entities': 'html.entities#module-html.entities', + 'html.parser': 'html.parser#module-html.parser', + 'http': 'http#module-http', + 'http.client': 'http.client#module-http.client', + 'http.cookiejar': 'http.cookiejar#module-http.cookiejar', + 'http.cookies': 'http.cookies#module-http.cookies', + 'http.server': 'http.server#module-http.server', + 'idlelib': 'idle#module-idlelib', + 'imaplib': 'imaplib#module-imaplib', + 'imghdr': 'imghdr#module-imghdr', + 'imp': 'imp#module-imp', + 'importlib': 'importlib#module-importlib', + 'importlib.abc': 'importlib#module-importlib.abc', + 'importlib.machinery': 'importlib#module-importlib.machinery', + 'importlib.metadata': 'importlib.metadata#module-importlib.metadata', + 'importlib.resources': 'importlib.resources#module-importlib.resources', + 'importlib.resources.abc': 'importlib.resources.abc#module-importlib.resources.abc', + 'importlib.util': 'importlib#module-importlib.util', + 'inspect': 'inspect#module-inspect', + 'io': 'io#module-io', + 'ipaddress': 'ipaddress#module-ipaddress', + 'itertools': 'itertools#module-itertools', + 'json': 'json#module-json', + 'json.tool': 'json#module-json.tool', + 'keyword': 'keyword#module-keyword', + 'linecache': 'linecache#module-linecache', + 'locale': 'locale#module-locale', + 'logging': 'logging#module-logging', + 'logging.config': 'logging.config#module-logging.config', + 'logging.handlers': 'logging.handlers#module-logging.handlers', + 'lzma': 'lzma#module-lzma', + 'mailbox': 'mailbox#module-mailbox', + 'mailcap': 'mailcap#module-mailcap', + 'marshal': 'marshal#module-marshal', + 'math': 'math#module-math', + 'mimetypes': 'mimetypes#module-mimetypes', + 'mmap': 'mmap#module-mmap', + 'modulefinder': 'modulefinder#module-modulefinder', + 'msilib': 'msilib#module-msilib', + 'msvcrt': 'msvcrt#module-msvcrt', + 'multiprocessing': 'multiprocessing#module-multiprocessing', + 'multiprocessing.connection': 'multiprocessing#module-multiprocessing.connection', + 'multiprocessing.dummy': 'multiprocessing#module-multiprocessing.dummy', + 'multiprocessing.managers': 'multiprocessing#module-multiprocessing.managers', + 'multiprocessing.pool': 'multiprocessing#module-multiprocessing.pool', + 'multiprocessing.shared_memory': 'multiprocessing.shared_memory#module-multiprocessing.shared_memory', + 'multiprocessing.sharedctypes': 'multiprocessing#module-multiprocessing.sharedctypes', + 'netrc': 'netrc#module-netrc', + 'nis': 'nis#module-nis', + 'nntplib': 'nntplib#module-nntplib', + 'numbers': 'numbers#module-numbers', + 'operator': 'operator#module-operator', + 'optparse': 'optparse#module-optparse', + 'os': 'os#module-os', + 'os.path': 'os.path#module-os.path', + 'ossaudiodev': 'ossaudiodev#module-ossaudiodev', + 'pathlib': 'pathlib#module-pathlib', + 'pathlib.types': 'pathlib#module-pathlib.types', + 'pdb': 'pdb#module-pdb', + 'pickle': 'pickle#module-pickle', + 'pickletools': 'pickletools#module-pickletools', + 'pipes': 'pipes#module-pipes', + 'pkgutil': 'pkgutil#module-pkgutil', + 'platform': 'platform#module-platform', + 'plistlib': 'plistlib#module-plistlib', + 'poplib': 'poplib#module-poplib', + 'posix': 'posix#module-posix', + 'pprint': 'pprint#module-pprint', + 'profile': 'profile#module-profile', + 'pstats': 'profile#module-pstats', + 'pty': 'pty#module-pty', + 'pwd': 'pwd#module-pwd', + 'py_compile': 'py_compile#module-py_compile', + 'pyclbr': 'pyclbr#module-pyclbr', + 'pydoc': 'pydoc#module-pydoc', + 'queue': 'queue#module-queue', + 'quopri': 'quopri#module-quopri', + 'random': 'random#module-random', + 're': 're#module-re', + 'readline': 'readline#module-readline', + 'reprlib': 'reprlib#module-reprlib', + 'resource': 'resource#module-resource', + 'rlcompleter': 'rlcompleter#module-rlcompleter', + 'runpy': 'runpy#module-runpy', + 'sched': 'sched#module-sched', + 'secrets': 'secrets#module-secrets', + 'select': 'select#module-select', + 'selectors': 'selectors#module-selectors', + 'shelve': 'shelve#module-shelve', + 'shlex': 'shlex#module-shlex', + 'shutil': 'shutil#module-shutil', + 'signal': 'signal#module-signal', + 'site': 'site#module-site', + 'sitecustomize': 'site#module-sitecustomize', + 'smtpd': 'smtpd#module-smtpd', + 'smtplib': 'smtplib#module-smtplib', + 'sndhdr': 'sndhdr#module-sndhdr', + 'socket': 'socket#module-socket', + 'socketserver': 'socketserver#module-socketserver', + 'spwd': 'spwd#module-spwd', + 'sqlite3': 'sqlite3#module-sqlite3', + 'ssl': 'ssl#module-ssl', + 'stat': 'stat#module-stat', + 'statistics': 'statistics#module-statistics', + 'string': 'string#module-string', + 'string.templatelib': 'string.templatelib#module-string.templatelib', + 'stringprep': 'stringprep#module-stringprep', + 'struct': 'struct#module-struct', + 'subprocess': 'subprocess#module-subprocess', + 'sunau': 'sunau#module-sunau', + 'symtable': 'symtable#module-symtable', + 'sys': 'sys#module-sys', + 'sys.monitoring': 'sys.monitoring#module-sys.monitoring', + 'sysconfig': 'sysconfig#module-sysconfig', + 'syslog': 'syslog#module-syslog', + 'tabnanny': 'tabnanny#module-tabnanny', + 'tarfile': 'tarfile#module-tarfile', + 'telnetlib': 'telnetlib#module-telnetlib', + 'tempfile': 'tempfile#module-tempfile', + 'termios': 'termios#module-termios', + 'test': 'test#module-test', + 'test.regrtest': 'test#module-test.regrtest', + 'test.support': 'test#module-test.support', + 'test.support.bytecode_helper': 'test#module-test.support.bytecode_helper', + 'test.support.import_helper': 'test#module-test.support.import_helper', + 'test.support.os_helper': 'test#module-test.support.os_helper', + 'test.support.script_helper': 'test#module-test.support.script_helper', + 'test.support.socket_helper': 'test#module-test.support.socket_helper', + 'test.support.threading_helper': 'test#module-test.support.threading_helper', + 'test.support.warnings_helper': 'test#module-test.support.warnings_helper', + 'textwrap': 'textwrap#module-textwrap', + 'threading': 'threading#module-threading', + 'time': 'time#module-time', + 'timeit': 'timeit#module-timeit', + 'tkinter': 'tkinter#module-tkinter', + 'tkinter.colorchooser': 'tkinter.colorchooser#module-tkinter.colorchooser', + 'tkinter.commondialog': 'dialog#module-tkinter.commondialog', + 'tkinter.dnd': 'tkinter.dnd#module-tkinter.dnd', + 'tkinter.filedialog': 'dialog#module-tkinter.filedialog', + 'tkinter.font': 'tkinter.font#module-tkinter.font', + 'tkinter.messagebox': 'tkinter.messagebox#module-tkinter.messagebox', + 'tkinter.scrolledtext': 'tkinter.scrolledtext#module-tkinter.scrolledtext', + 'tkinter.simpledialog': 'dialog#module-tkinter.simpledialog', + 'tkinter.ttk': 'tkinter.ttk#module-tkinter.ttk', + 'token': 'token#module-token', + 'tokenize': 'tokenize#module-tokenize', + 'tomllib': 'tomllib#module-tomllib', + 'trace': 'trace#module-trace', + 'traceback': 'traceback#module-traceback', + 'tracemalloc': 'tracemalloc#module-tracemalloc', + 'tty': 'tty#module-tty', + 'turtle': 'turtle#module-turtle', + 'turtledemo': 'turtle#module-turtledemo', + 'types': 'types#module-types', + 'typing': 'typing#module-typing', + 'unicodedata': 'unicodedata#module-unicodedata', + 'unittest': 'unittest#module-unittest', + 'unittest.mock': 'unittest.mock#module-unittest.mock', + 'urllib': 'urllib#module-urllib', + 'urllib.error': 'urllib.error#module-urllib.error', + 'urllib.parse': 'urllib.parse#module-urllib.parse', + 'urllib.request': 'urllib.request#module-urllib.request', + 'urllib.response': 'urllib.request#module-urllib.response', + 'urllib.robotparser': 'urllib.robotparser#module-urllib.robotparser', + 'usercustomize': 'site#module-usercustomize', + 'uu': 'uu#module-uu', + 'uuid': 'uuid#module-uuid', + 'venv': 'venv#module-venv', + 'warnings': 'warnings#module-warnings', + 'wave': 'wave#module-wave', + 'weakref': 'weakref#module-weakref', + 'webbrowser': 'webbrowser#module-webbrowser', + 'winreg': 'winreg#module-winreg', + 'winsound': 'winsound#module-winsound', + 'wsgiref': 'wsgiref#module-wsgiref', + 'wsgiref.handlers': 'wsgiref#module-wsgiref.handlers', + 'wsgiref.headers': 'wsgiref#module-wsgiref.headers', + 'wsgiref.simple_server': 'wsgiref#module-wsgiref.simple_server', + 'wsgiref.types': 'wsgiref#module-wsgiref.types', + 'wsgiref.util': 'wsgiref#module-wsgiref.util', + 'wsgiref.validate': 'wsgiref#module-wsgiref.validate', + 'xdrlib': 'xdrlib#module-xdrlib', + 'xml': 'xml#module-xml', + 'xml.dom': 'xml.dom#module-xml.dom', + 'xml.dom.minidom': 'xml.dom.minidom#module-xml.dom.minidom', + 'xml.dom.pulldom': 'xml.dom.pulldom#module-xml.dom.pulldom', + 'xml.etree.ElementInclude': 'xml.etree.elementtree#module-xml.etree.ElementInclude', + 'xml.etree.ElementTree': 'xml.etree.elementtree#module-xml.etree.ElementTree', + 'xml.parsers.expat': 'pyexpat#module-xml.parsers.expat', + 'xml.parsers.expat.errors': 'pyexpat#module-xml.parsers.expat.errors', + 'xml.parsers.expat.model': 'pyexpat#module-xml.parsers.expat.model', + 'xml.sax': 'xml.sax#module-xml.sax', + 'xml.sax.handler': 'xml.sax.handler#module-xml.sax.handler', + 'xml.sax.saxutils': 'xml.sax.utils#module-xml.sax.saxutils', + 'xml.sax.xmlreader': 'xml.sax.reader#module-xml.sax.xmlreader', + 'xmlrpc': 'xmlrpc#module-xmlrpc', + 'xmlrpc.client': 'xmlrpc.client#module-xmlrpc.client', + 'xmlrpc.server': 'xmlrpc.server#module-xmlrpc.server', + 'zipapp': 'zipapp#module-zipapp', + 'zipfile': 'zipfile#module-zipfile', + 'zipimport': 'zipimport#module-zipimport', + 'zlib': 'zlib#module-zlib', + 'zoneinfo': 'zoneinfo#module-zoneinfo', +} diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index 56317b8a724..4e31cf08bb5 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,4 +1,4 @@ -# Autogenerated by Sphinx on Fri Dec 5 18:49:09 2025 +# Autogenerated by Sphinx on Tue Feb 3 17:32:13 2026 # as part of the release process. topics = { @@ -2000,7 +2000,7 @@ class attributes; they are shared by instances. Instance attributes ... except* BlockingIOError as e: ... print(repr(e)) ... - ExceptionGroup('', (BlockingIOError())) + ExceptionGroup('', (BlockingIOError(),)) "break", "continue" and "return" cannot appear in an "except*" clause. @@ -5796,7 +5796,9 @@ class of the instance or a *non-virtual base class* thereof. The | | With no precision given, uses a precision of "6" digits | | | after the decimal point for "float", and shows all | | | coefficient digits for "Decimal". If "p=0", the decimal | - | | point is omitted unless the "#" option is used. | + | | point is omitted unless the "#" option is used. For | + | | "float", the exponent always contains at least two digits, | + | | and is zero if the value is zero. | +-----------+------------------------------------------------------------+ | "'E'" | Scientific notation. Same as "'e'" except it uses an upper | | | case ‘E’ as the separator character. | @@ -9830,7 +9832,12 @@ class is used in a class pattern with positional arguments, each it is intended to remove all case distinctions in a string. For example, the German lowercase letter "'ß'" is equivalent to ""ss"". Since it is already lowercase, "lower()" would do nothing to "'ß'"; - "casefold()" converts it to ""ss"". + "casefold()" converts it to ""ss"". For example: + + >>> 'straße'.lower() + 'straße' + >>> 'straße'.casefold() + 'strasse' The casefolding algorithm is described in section 3.13 ‘Default Case Folding’ of the Unicode Standard. @@ -10019,7 +10026,18 @@ class is used in a class pattern with positional arguments, each str.index(sub[, start[, end]]) Like "find()", but raise "ValueError" when the substring is not - found. + found. For example: + + >>> 'spam, spam, spam'.index('spam') + 0 + >>> 'spam, spam, spam'.index('eggs') + Traceback (most recent call last): + File "", line 1, in + 'spam, spam, spam'.index('eggs') + ~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^ + ValueError: substring not found + + See also "rindex()". str.isalnum() @@ -10118,7 +10136,19 @@ class is used in a class pattern with positional arguments, each that have the Unicode numeric value property, e.g. U+2155, VULGAR FRACTION ONE FIFTH. Formally, numeric characters are those with the property value Numeric_Type=Digit, Numeric_Type=Decimal or - Numeric_Type=Numeric. + Numeric_Type=Numeric. For example: + + >>> '0123456789'.isnumeric() + True + >>> '٠١٢٣٤٥٦٧٨٩'.isnumeric() # Arabic-indic digit zero to nine + True + >>> '⅕'.isnumeric() # Vulgar fraction one fifth + True + >>> '²'.isdecimal(), '²'.isdigit(), '²'.isnumeric() + (False, True, True) + + See also "isdecimal()" and "isdigit()". Numeric characters are a + superset of decimal numbers. str.isprintable() @@ -10136,6 +10166,13 @@ class is used in a class pattern with positional arguments, each plus the ASCII space 0x20. Nonprintable characters are those in group Separator or Other (Z or C), except the ASCII space. + For example: + + >>> ''.isprintable(), ' '.isprintable() + (True, True) + >>> '\t'.isprintable(), '\n'.isprintable() + (False, False) + str.isspace() Return "True" if there are only whitespace characters in the string @@ -10201,10 +10238,24 @@ class is used in a class pattern with positional arguments, each space). The original string is returned if *width* is less than or equal to "len(s)". + For example: + + >>> 'Python'.ljust(10) + 'Python ' + >>> 'Python'.ljust(10, '.') + 'Python....' + >>> 'Monty Python'.ljust(10, '.') + 'Monty Python' + + See also "rjust()". + str.lower() Return a copy of the string with all the cased characters [4] - converted to lowercase. + converted to lowercase. For example: + + >>> 'Lower Method Example'.lower() + 'lower method example' The lowercasing algorithm used is described in section 3.13 ‘Default Case Folding’ of the Unicode Standard. @@ -10268,6 +10319,8 @@ class is used in a class pattern with positional arguments, each Added in version 3.9. + See also "removesuffix()" and "startswith()". + str.removesuffix(suffix, /) If the string ends with the *suffix* string and that *suffix* is @@ -10281,12 +10334,19 @@ class is used in a class pattern with positional arguments, each Added in version 3.9. + See also "removeprefix()" and "endswith()". + str.replace(old, new, /, count=-1) Return a copy of the string with all occurrences of substring *old* replaced by *new*. If *count* is given, only the first *count* occurrences are replaced. If *count* is not specified or "-1", then - all occurrences are replaced. + all occurrences are replaced. For example: + + >>> 'spam, spam, spam'.replace('spam', 'eggs') + 'eggs, eggs, eggs' + >>> 'spam, spam, spam'.replace('spam', 'eggs', 1) + 'eggs, spam, spam' Changed in version 3.13: *count* is now supported as a keyword argument. @@ -10296,12 +10356,30 @@ class is used in a class pattern with positional arguments, each Return the highest index in the string where substring *sub* is found, such that *sub* is contained within "s[start:end]". Optional arguments *start* and *end* are interpreted as in slice - notation. Return "-1" on failure. + notation. Return "-1" on failure. For example: + + >>> 'spam, spam, spam'.rfind('sp') + 12 + >>> 'spam, spam, spam'.rfind('sp', 0, 10) + 6 + + See also "find()" and "rindex()". str.rindex(sub[, start[, end]]) Like "rfind()" but raises "ValueError" when the substring *sub* is - not found. + not found. For example: + + >>> 'spam, spam, spam'.rindex('spam') + 12 + >>> 'spam, spam, spam'.rindex('eggs') + Traceback (most recent call last): + File "", line 1, in + 'spam, spam, spam'.rindex('eggs') + ~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^ + ValueError: substring not found + + See also "index()" and "find()". str.rjust(width, fillchar=' ', /) @@ -10318,6 +10396,17 @@ class is used in a class pattern with positional arguments, each found, return a 3-tuple containing two empty strings, followed by the string itself. + For example: + + >>> 'Monty Python'.rpartition(' ') + ('Monty', ' ', 'Python') + >>> "Monty Python's Flying Circus".rpartition(' ') + ("Monty Python's Flying", ' ', 'Circus') + >>> 'Monty Python'.rpartition('-') + ('', '', 'Monty Python') + + See also "partition()". + str.rsplit(sep=None, maxsplit=-1) Return a list of the words in the string, using *sep* as the @@ -11088,7 +11177,7 @@ class is used in a class pattern with positional arguments, each * In "fstring_replacement_field", if "f_debug_specifier" is present, all whitespace after the opening brace until the - "f_debug_specifier", as well as whitespace immediatelly following + "f_debug_specifier", as well as whitespace immediately following "f_debug_specifier", is retained as part of the expression. **CPython implementation detail:** The expression is not handled in @@ -11211,8 +11300,11 @@ class is used in a class pattern with positional arguments, each By default, an object is considered true unless its class defines either a "__bool__()" method that returns "False" or a "__len__()" -method that returns zero, when called with the object. [1] Here are -most of the built-in objects considered false: +method that returns zero, when called with the object. [1] If one of +the methods raises an exception when called, the exception is +propagated and the object does not have a truth value (for example, +"NotImplemented"). Here are most of the built-in objects considered +false: * constants defined to be false: "None" and "False" @@ -11393,7 +11485,7 @@ class is used in a class pattern with positional arguments, each ... except* BlockingIOError as e: ... print(repr(e)) ... - ExceptionGroup('', (BlockingIOError())) + ExceptionGroup('', (BlockingIOError(),)) "break", "continue" and "return" cannot appear in an "except*" clause. @@ -11777,6 +11869,10 @@ def foo(): +----------------------------------------------------+----------------------------------------------------+ | Attribute | Meaning | |====================================================|====================================================| +| function.__builtins__ | A reference to the "dictionary" that holds the | +| | function’s builtins namespace. Added in version | +| | 3.10. | ++----------------------------------------------------+----------------------------------------------------+ | function.__globals__ | A reference to the "dictionary" that holds the | | | function’s global variables – the global namespace | | | of the module in which the function was defined. | @@ -12871,10 +12967,6 @@ class dict(iterable, /, **kwargs) the keyword argument replaces the value from the positional argument. - Providing keyword arguments as in the first example only works for - keys that are valid Python identifiers. Otherwise, any valid keys - can be used. - Dictionaries compare equal if and only if they have the same "(key, value)" pairs (regardless of ordering). Order comparisons (‘<’, ‘<=’, ‘>=’, ‘>’) raise "TypeError". To illustrate dictionary @@ -13387,10 +13479,17 @@ class dict(iterable, /, **kwargs) note that "-0" is still "0". 4. The slice of *s* from *i* to *j* is defined as the sequence of - items with index *k* such that "i <= k < j". If *i* or *j* is - greater than "len(s)", use "len(s)". If *i* is omitted or "None", - use "0". If *j* is omitted or "None", use "len(s)". If *i* is - greater than or equal to *j*, the slice is empty. + items with index *k* such that "i <= k < j". + + * If *i* is omitted or "None", use "0". + + * If *j* is omitted or "None", use "len(s)". + + * If *i* or *j* is less than "-len(s)", use "0". + + * If *i* or *j* is greater than "len(s)", use "len(s)". + + * If *i* is greater than or equal to *j*, the slice is empty. 5. The slice of *s* from *i* to *j* with step *k* is defined as the sequence of items with index "x = i + n*k" such that "0 <= n < @@ -13655,6 +13754,76 @@ class list(iterable=(), /) empty for the duration, and raises "ValueError" if it can detect that the list has been mutated during a sort. +Thread safety: Reading a single element from a "list" is *atomic*: + + lst[i] # list.__getitem__ + +The following methods traverse the list and use *atomic* reads of each +item to perform their function. That means that they may return +results affected by concurrent modifications: + + item in lst + lst.index(item) + lst.count(item) + +All of the above methods/operations are also lock-free. They do not +block concurrent modifications. Other operations that hold a lock will +not block these from observing intermediate states.All other +operations from here on block using the per-object lock.Writing a +single item via "lst[i] = x" is safe to call from multiple threads and +will not corrupt the list.The following operations return new objects +and appear *atomic* to other threads: + + lst1 + lst2 # concatenates two lists into a new list + x * lst # repeats lst x times into a new list + lst.copy() # returns a shallow copy of the list + +Methods that only operate on a single elements with no shifting +required are *atomic*: + + lst.append(x) # append to the end of the list, no shifting required + lst.pop() # pop element from the end of the list, no shifting required + +The "clear()" method is also *atomic*. Other threads cannot observe +elements being removed.The "sort()" method is not *atomic*. Other +threads cannot observe intermediate states during sorting, but the +list appears empty for the duration of the sort.The following +operations may allow lock-free operations to observe intermediate +states since they modify multiple elements in place: + + lst.insert(idx, item) # shifts elements + lst.pop(idx) # idx not at the end of the list, shifts elements + lst *= x # copies elements in place + +The "remove()" method may allow concurrent modifications since element +comparison may execute arbitrary Python code (via +"__eq__()")."extend()" is safe to call from multiple threads. +However, its guarantees depend on the iterable passed to it. If it is +a "list", a "tuple", a "set", a "frozenset", a "dict" or a dictionary +view object (but not their subclasses), the "extend" operation is safe +from concurrent modifications to the iterable. Otherwise, an iterator +is created which can be concurrently modified by another thread. The +same applies to inplace concatenation of a list with other iterables +when using "lst += iterable".Similarly, assigning to a list slice with +"lst[i:j] = iterable" is safe to call from multiple threads, but +"iterable" is only locked when it is also a "list" (but not its +subclasses).Operations that involve multiple accesses, as well as +iteration, are never atomic. For example: + + # NOT atomic: read-modify-write + lst[i] = lst[i] + 1 + + # NOT atomic: check-then-act + if lst: + item = lst.pop() + + # NOT thread-safe: iteration while modifying + for item in lst: + process(item) # another thread may modify lst + +Consider external synchronization when sharing "list" instances across +threads. See Python support for free threading for more information. + Tuples ====== diff --git a/Lib/queue.py b/Lib/queue.py index 25beb46e30d..c0b35987654 100644 --- a/Lib/queue.py +++ b/Lib/queue.py @@ -80,9 +80,6 @@ def task_done(self): have been processed (meaning that a task_done() call was received for every item that had been put() into the queue). - shutdown(immediate=True) calls task_done() for each remaining item in - the queue. - Raises a ValueError if called more times than there were items placed in the queue. ''' @@ -239,9 +236,11 @@ def shutdown(self, immediate=False): By default, gets will only raise once the queue is empty. Set 'immediate' to True to make gets raise immediately instead. - All blocked callers of put() and get() will be unblocked. If - 'immediate', a task is marked as done for each item remaining in - the queue, which may unblock callers of join(). + All blocked callers of put() and get() will be unblocked. + + If 'immediate', the queue is drained and unfinished tasks + is reduced by the number of drained tasks. If unfinished tasks + is reduced to zero, callers of Queue.join are unblocked. ''' with self.mutex: self.is_shutdown = True diff --git a/Lib/random.py b/Lib/random.py index 36e3925811c..86d562f0b8a 100644 --- a/Lib/random.py +++ b/Lib/random.py @@ -50,39 +50,18 @@ # Adrian Baddeley. Adapted by Raymond Hettinger for use with # the Mersenne Twister and os.urandom() core generators. -from warnings import warn as _warn from math import log as _log, exp as _exp, pi as _pi, e as _e, ceil as _ceil from math import sqrt as _sqrt, acos as _acos, cos as _cos, sin as _sin from math import tau as TWOPI, floor as _floor, isfinite as _isfinite from math import lgamma as _lgamma, fabs as _fabs, log2 as _log2 -try: - from os import urandom as _urandom -except ImportError: - # XXX RUSTPYTHON - # On wasm, _random.Random.random() does give a proper random value, but - # we don't have the os module - def _urandom(*args, **kwargs): - raise NotImplementedError("urandom") - _os = None -from _collections_abc import Set as _Set, Sequence as _Sequence +from os import urandom as _urandom +from _collections_abc import Sequence as _Sequence from operator import index as _index from itertools import accumulate as _accumulate, repeat as _repeat from bisect import bisect as _bisect -try: - import os as _os -except ImportError: - # XXX RUSTPYTHON - # On wasm, we don't have the os module - _os = None +import os as _os import _random -try: - # hashlib is pretty heavy to load, try lean internal module first - from _sha2 import sha512 as _sha512 -except ImportError: - # fallback to official implementation - from hashlib import sha512 as _sha512 - __all__ = [ "Random", "SystemRandom", @@ -118,6 +97,7 @@ def _urandom(*args, **kwargs): BPF = 53 # Number of bits in a float RECIP_BPF = 2 ** -BPF _ONE = 1 +_sha512 = None class Random(_random.Random): @@ -172,13 +152,23 @@ def seed(self, a=None, version=2): a = -2 if x == -1 else x elif version == 2 and isinstance(a, (str, bytes, bytearray)): + global _sha512 + if _sha512 is None: + try: + # hashlib is pretty heavy to load, try lean internal + # module first + from _sha2 import sha512 as _sha512 + except ImportError: + # fallback to official implementation + from hashlib import sha512 as _sha512 + if isinstance(a, str): a = a.encode() a = int.from_bytes(a + _sha512(a).digest()) elif not isinstance(a, (type(None), int, float, str, bytes, bytearray)): - raise TypeError('The only supported seed types are: None,\n' - 'int, float, str, bytes, and bytearray.') + raise TypeError('The only supported seed types are:\n' + 'None, int, float, str, bytes, and bytearray.') super().seed(a) self.gauss_next = None @@ -255,11 +245,10 @@ def __init_subclass__(cls, /, **kwargs): def _randbelow_with_getrandbits(self, n): "Return a random int in the range [0,n). Defined for n > 0." - getrandbits = self.getrandbits k = n.bit_length() - r = getrandbits(k) # 0 <= r < 2**k + r = self.getrandbits(k) # 0 <= r < 2**k while r >= n: - r = getrandbits(k) + r = self.getrandbits(k) return r def _randbelow_without_getrandbits(self, n, maxsize=1<= maxsize: - _warn("Underlying random() generator does not supply \n" - "enough bits to choose from a population range this large.\n" - "To remove the range limitation, add a getrandbits() method.") + from warnings import warn + warn("Underlying random() generator does not supply \n" + "enough bits to choose from a population range this large.\n" + "To remove the range limitation, add a getrandbits() method.") return _floor(random() * n) rem = maxsize % n limit = (maxsize - rem) / maxsize # int(limit * maxsize) % n == 0 @@ -345,8 +335,11 @@ def randrange(self, start, stop=None, step=_ONE): def randint(self, a, b): """Return random integer in range [a, b], including both end points. """ - - return self.randrange(a, b+1) + a = _index(a) + b = _index(b) + if b < a: + raise ValueError(f"empty range in randint({a}, {b})") + return a + self._randbelow(b - a + 1) ## -------------------- sequence methods ------------------- @@ -430,11 +423,11 @@ def sample(self, population, k, *, counts=None): cum_counts = list(_accumulate(counts)) if len(cum_counts) != n: raise ValueError('The number of counts does not match the population') - total = cum_counts.pop() + total = cum_counts.pop() if cum_counts else 0 if not isinstance(total, int): raise TypeError('Counts must be integers') - if total <= 0: - raise ValueError('Total of counts must be greater than zero') + if total < 0: + raise ValueError('Counts must be non-negative') selections = self.sample(range(total), k=k) bisect = _bisect return [population[bisect(cum_counts, s)] for s in selections] @@ -801,12 +794,18 @@ def binomialvariate(self, n=1, p=0.5): sum(random() < p for i in range(n)) - Returns an integer in the range: 0 <= X <= n + Returns an integer in the range: + + 0 <= X <= n + + The integer is chosen with the probability: + + P(X == k) = math.comb(n, k) * p ** k * (1 - p) ** (n - k) The mean (expected value) and variance of the random variable are: E[X] = n * p - Var[x] = n * p * (1 - p) + Var[X] = n * p * (1 - p) """ # Error check inputs and handle edge cases @@ -1005,5 +1004,75 @@ def _test(N=10_000): _os.register_at_fork(after_in_child=_inst.seed) +# ------------------------------------------------------ +# -------------- command-line interface ---------------- + + +def _parse_args(arg_list: list[str] | None): + import argparse + parser = argparse.ArgumentParser( + formatter_class=argparse.RawTextHelpFormatter, color=True) + group = parser.add_mutually_exclusive_group() + group.add_argument( + "-c", "--choice", nargs="+", + help="print a random choice") + group.add_argument( + "-i", "--integer", type=int, metavar="N", + help="print a random integer between 1 and N inclusive") + group.add_argument( + "-f", "--float", type=float, metavar="N", + help="print a random floating-point number between 0 and N inclusive") + group.add_argument( + "--test", type=int, const=10_000, nargs="?", + help=argparse.SUPPRESS) + parser.add_argument("input", nargs="*", + help="""\ +if no options given, output depends on the input + string or multiple: same as --choice + integer: same as --integer + float: same as --float""") + args = parser.parse_args(arg_list) + return args, parser.format_help() + + +def main(arg_list: list[str] | None = None) -> int | str: + args, help_text = _parse_args(arg_list) + + # Explicit arguments + if args.choice: + return choice(args.choice) + + if args.integer is not None: + return randint(1, args.integer) + + if args.float is not None: + return uniform(0, args.float) + + if args.test: + _test(args.test) + return "" + + # No explicit argument, select based on input + if len(args.input) == 1: + val = args.input[0] + try: + # Is it an integer? + val = int(val) + return randint(1, val) + except ValueError: + try: + # Is it a float? + val = float(val) + return uniform(0, val) + except ValueError: + # Split in case of space-separated string: "a b c" + return choice(val.split()) + + if len(args.input) >= 2: + return choice(args.input) + + return help_text + + if __name__ == '__main__': - _test() + print(main()) diff --git a/Lib/shlex.py b/Lib/shlex.py index f4821616b62..5959f52dd12 100644 --- a/Lib/shlex.py +++ b/Lib/shlex.py @@ -7,11 +7,7 @@ # iterator interface by Gustavo Niemeyer, April 2003. # changes to tokenize more like Posix shells by Vinay Sajip, July 2016. -import os -import re import sys -from collections import deque - from io import StringIO __all__ = ["shlex", "split", "quote", "join"] @@ -20,6 +16,8 @@ class shlex: "A lexical analyzer class for simple shell-like syntaxes." def __init__(self, instream=None, infile=None, posix=False, punctuation_chars=False): + from collections import deque # deferred import for performance + if isinstance(instream, str): instream = StringIO(instream) if instream is not None: @@ -278,6 +276,7 @@ def read_token(self): def sourcehook(self, newfile): "Hook called on a filename to be sourced." + import os.path if newfile[0] == '"': newfile = newfile[1:-1] # This implements cpp-like semantics for relative-path inclusion. @@ -318,13 +317,20 @@ def join(split_command): return ' '.join(quote(arg) for arg in split_command) -_find_unsafe = re.compile(r'[^\w@%+=:,./-]', re.ASCII).search - def quote(s): """Return a shell-escaped version of the string *s*.""" if not s: return "''" - if _find_unsafe(s) is None: + + if not isinstance(s, str): + raise TypeError(f"expected string object, got {type(s).__name__!r}") + + # Use bytes.translate() for performance + safe_chars = (b'%+,-./0123456789:=@' + b'ABCDEFGHIJKLMNOPQRSTUVWXYZ_' + b'abcdefghijklmnopqrstuvwxyz') + # No quoting is needed if `s` is an ASCII string consisting only of `safe_chars` + if s.isascii() and not s.encode().translate(None, delete=safe_chars): return s # use single quotes, and put single quotes into double quotes diff --git a/Lib/shutil.py b/Lib/shutil.py index 7df972012c7..8d8fe145567 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -32,6 +32,13 @@ except ImportError: _LZMA_SUPPORTED = False +try: + from compression import zstd + del zstd + _ZSTD_SUPPORTED = True +except ImportError: + _ZSTD_SUPPORTED = False + _WINDOWS = os.name == 'nt' posix = nt = None if os.name == 'posix': @@ -44,11 +51,12 @@ else: _winapi = None -COPY_BUFSIZE = 1024 * 1024 if _WINDOWS else 64 * 1024 +COPY_BUFSIZE = 1024 * 1024 if _WINDOWS else 256 * 1024 # This should never be removed, see rationale in: # https://bugs.python.org/issue43743#msg393429 _USE_CP_SENDFILE = (hasattr(os, "sendfile") - and sys.platform.startswith(("linux", "android"))) + and sys.platform.startswith(("linux", "android", "sunos"))) +_USE_CP_COPY_FILE_RANGE = hasattr(os, "copy_file_range") _HAS_FCOPYFILE = posix and hasattr(posix, "_fcopyfile") # macOS # CMD defaults in Windows 10 @@ -56,7 +64,7 @@ __all__ = ["copyfileobj", "copyfile", "copymode", "copystat", "copy", "copy2", "copytree", "move", "rmtree", "Error", "SpecialFileError", - "ExecError", "make_archive", "get_archive_formats", + "make_archive", "get_archive_formats", "register_archive_format", "unregister_archive_format", "get_unpack_formats", "register_unpack_format", "unregister_unpack_format", "unpack_archive", @@ -74,8 +82,6 @@ class SpecialFileError(OSError): """Raised when trying to do a kind of operation (e.g. copying) which is not supported on a special file (e.g. a named pipe)""" -class ExecError(OSError): - """Raised when a command could not be executed""" class ReadError(OSError): """Raised when an archive cannot be read""" @@ -109,10 +115,70 @@ def _fastcopy_fcopyfile(fsrc, fdst, flags): else: raise err from None +def _determine_linux_fastcopy_blocksize(infd): + """Determine blocksize for fastcopying on Linux. + + Hopefully the whole file will be copied in a single call. + The copying itself should be performed in a loop 'till EOF is + reached (0 return) so a blocksize smaller or bigger than the actual + file size should not make any difference, also in case the file + content changes while being copied. + """ + try: + blocksize = max(os.fstat(infd).st_size, 2 ** 23) # min 8 MiB + except OSError: + blocksize = 2 ** 27 # 128 MiB + # On 32-bit architectures truncate to 1 GiB to avoid OverflowError, + # see gh-82500. + if sys.maxsize < 2 ** 32: + blocksize = min(blocksize, 2 ** 30) + return blocksize + +def _fastcopy_copy_file_range(fsrc, fdst): + """Copy data from one regular mmap-like fd to another by using + a high-performance copy_file_range(2) syscall that gives filesystems + an opportunity to implement the use of reflinks or server-side copy. + + This should work on Linux >= 4.5 only. + """ + try: + infd = fsrc.fileno() + outfd = fdst.fileno() + except Exception as err: + raise _GiveupOnFastCopy(err) # not a regular file + + blocksize = _determine_linux_fastcopy_blocksize(infd) + offset = 0 + while True: + try: + n_copied = os.copy_file_range(infd, outfd, blocksize, offset_dst=offset) + except OSError as err: + # ...in oder to have a more informative exception. + err.filename = fsrc.name + err.filename2 = fdst.name + + if err.errno == errno.ENOSPC: # filesystem is full + raise err from None + + # Give up on first call and if no data was copied. + if offset == 0 and os.lseek(outfd, 0, os.SEEK_CUR) == 0: + raise _GiveupOnFastCopy(err) + + raise err + else: + if n_copied == 0: + # If no bytes have been copied yet, copy_file_range + # might silently fail. + # https://lore.kernel.org/linux-fsdevel/20210126233840.GG4626@dread.disaster.area/T/#m05753578c7f7882f6e9ffe01f981bc223edef2b0 + if offset == 0: + raise _GiveupOnFastCopy() + break + offset += n_copied + def _fastcopy_sendfile(fsrc, fdst): """Copy data from one regular mmap-like fd to another by using high-performance sendfile(2) syscall. - This should work on Linux >= 2.6.33 only. + This should work on Linux >= 2.6.33, Android and Solaris. """ # Note: copyfileobj() is left alone in order to not introduce any # unexpected breakage. Possible risks by using zero-copy calls @@ -130,26 +196,13 @@ def _fastcopy_sendfile(fsrc, fdst): except Exception as err: raise _GiveupOnFastCopy(err) # not a regular file - # Hopefully the whole file will be copied in a single call. - # sendfile() is called in a loop 'till EOF is reached (0 return) - # so a bufsize smaller or bigger than the actual file size - # should not make any difference, also in case the file content - # changes while being copied. - try: - blocksize = max(os.fstat(infd).st_size, 2 ** 23) # min 8MiB - except OSError: - blocksize = 2 ** 27 # 128MiB - # On 32-bit architectures truncate to 1GiB to avoid OverflowError, - # see bpo-38319. - if sys.maxsize < 2 ** 32: - blocksize = min(blocksize, 2 ** 30) - + blocksize = _determine_linux_fastcopy_blocksize(infd) offset = 0 while True: try: sent = os.sendfile(outfd, infd, offset, blocksize) except OSError as err: - # ...in oder to have a more informative exception. + # ...in order to have a more informative exception. err.filename = fsrc.name err.filename2 = fdst.name @@ -267,13 +320,21 @@ def copyfile(src, dst, *, follow_symlinks=True): return dst except _GiveupOnFastCopy: pass - # Linux - elif _USE_CP_SENDFILE: - try: - _fastcopy_sendfile(fsrc, fdst) - return dst - except _GiveupOnFastCopy: - pass + # Linux / Android / Solaris + elif _USE_CP_SENDFILE or _USE_CP_COPY_FILE_RANGE: + # reflink may be implicit in copy_file_range. + if _USE_CP_COPY_FILE_RANGE: + try: + _fastcopy_copy_file_range(fsrc, fdst) + return dst + except _GiveupOnFastCopy: + pass + if _USE_CP_SENDFILE: + try: + _fastcopy_sendfile(fsrc, fdst) + return dst + except _GiveupOnFastCopy: + pass # Windows, see: # https://github.com/python/cpython/pull/7160#discussion_r195405230 elif _WINDOWS and file_size > 0: @@ -605,7 +666,22 @@ def _rmtree_islink(st): return stat.S_ISLNK(st.st_mode) # version vulnerable to race conditions -def _rmtree_unsafe(path, onexc): +def _rmtree_unsafe(path, dir_fd, onexc): + if dir_fd is not None: + raise NotImplementedError("dir_fd unavailable on this platform") + try: + st = os.lstat(path) + except OSError as err: + onexc(os.lstat, path, err) + return + try: + if _rmtree_islink(st): + # symlinks to directories are forbidden, see bug #1669 + raise OSError("Cannot call rmtree on a symbolic link") + except OSError as err: + onexc(os.path.islink, path, err) + # can't continue even if onexc hook returns + return def onerror(err): if not isinstance(err, FileNotFoundError): onexc(os.scandir, err.filename, err) @@ -635,7 +711,26 @@ def onerror(err): onexc(os.rmdir, path, err) # Version using fd-based APIs to protect against races -def _rmtree_safe_fd(stack, onexc): +def _rmtree_safe_fd(path, dir_fd, onexc): + # While the unsafe rmtree works fine on bytes, the fd based does not. + if isinstance(path, bytes): + path = os.fsdecode(path) + stack = [(os.lstat, dir_fd, path, None)] + try: + while stack: + _rmtree_safe_fd_step(stack, onexc) + finally: + # Close any file descriptors still on the stack. + while stack: + func, fd, path, entry = stack.pop() + if func is not os.close: + continue + try: + os.close(fd) + except OSError as err: + onexc(os.close, path, err) + +def _rmtree_safe_fd_step(stack, onexc): # Each stack item has four elements: # * func: The first operation to perform: os.lstat, os.close or os.rmdir. # Walking a directory starts with an os.lstat() to detect symlinks; in @@ -710,6 +805,7 @@ def _rmtree_safe_fd(stack, onexc): os.supports_dir_fd and os.scandir in os.supports_fd and os.stat in os.supports_follow_symlinks) +_rmtree_impl = _rmtree_safe_fd if _use_fd_functions else _rmtree_unsafe def rmtree(path, ignore_errors=False, onerror=None, *, onexc=None, dir_fd=None): """Recursively delete a directory tree. @@ -753,41 +849,7 @@ def onexc(*args): exc_info = type(exc), exc, exc.__traceback__ return onerror(func, path, exc_info) - if _use_fd_functions: - # While the unsafe rmtree works fine on bytes, the fd based does not. - if isinstance(path, bytes): - path = os.fsdecode(path) - stack = [(os.lstat, dir_fd, path, None)] - try: - while stack: - _rmtree_safe_fd(stack, onexc) - finally: - # Close any file descriptors still on the stack. - while stack: - func, fd, path, entry = stack.pop() - if func is not os.close: - continue - try: - os.close(fd) - except OSError as err: - onexc(os.close, path, err) - else: - if dir_fd is not None: - raise NotImplementedError("dir_fd unavailable on this platform") - try: - st = os.lstat(path) - except OSError as err: - onexc(os.lstat, path, err) - return - try: - if _rmtree_islink(st): - # symlinks to directories are forbidden, see bug #1669 - raise OSError("Cannot call rmtree on a symbolic link") - except OSError as err: - onexc(os.path.islink, path, err) - # can't continue even if onexc hook returns - return - return _rmtree_unsafe(path, onexc) + _rmtree_impl(path, dir_fd, onexc) # Allow introspection of whether or not the hardening against symlink # attacks is supported on the current platform @@ -932,14 +994,14 @@ def _make_tarball(base_name, base_dir, compress="gzip", verbose=0, dry_run=0, """Create a (possibly compressed) tar file from all the files under 'base_dir'. - 'compress' must be "gzip" (the default), "bzip2", "xz", or None. + 'compress' must be "gzip" (the default), "bzip2", "xz", "zst", or None. 'owner' and 'group' can be used to define an owner and a group for the archive that is being built. If not provided, the current owner and group will be used. The output tar file will be named 'base_name' + ".tar", possibly plus - the appropriate compression extension (".gz", ".bz2", or ".xz"). + the appropriate compression extension (".gz", ".bz2", ".xz", or ".zst"). Returns the output filename. """ @@ -951,6 +1013,8 @@ def _make_tarball(base_name, base_dir, compress="gzip", verbose=0, dry_run=0, tar_compression = 'bz2' elif _LZMA_SUPPORTED and compress == 'xz': tar_compression = 'xz' + elif _ZSTD_SUPPORTED and compress == 'zst': + tar_compression = 'zst' else: raise ValueError("bad value for 'compress', or compression format not " "supported : {0}".format(compress)) @@ -1079,6 +1143,10 @@ def _make_zipfile(base_name, base_dir, verbose=0, dry_run=0, _ARCHIVE_FORMATS['xztar'] = (_make_tarball, [('compress', 'xz')], "xz'ed tar-file") +if _ZSTD_SUPPORTED: + _ARCHIVE_FORMATS['zstdtar'] = (_make_tarball, [('compress', 'zst')], + "zstd'ed tar-file") + def get_archive_formats(): """Returns a list of supported formats for archiving and unarchiving. @@ -1119,7 +1187,7 @@ def make_archive(base_name, format, root_dir=None, base_dir=None, verbose=0, 'base_name' is the name of the file to create, minus any format-specific extension; 'format' is the archive format: one of "zip", "tar", "gztar", - "bztar", or "xztar". Or any other registered format. + "bztar", "xztar", or "zstdtar". Or any other registered format. 'root_dir' is a directory that will be the root directory of the archive; ie. we typically chdir into 'root_dir' before creating the @@ -1269,7 +1337,7 @@ def _unpack_zipfile(filename, extract_dir): zip.close() def _unpack_tarfile(filename, extract_dir, *, filter=None): - """Unpack tar/tar.gz/tar.bz2/tar.xz `filename` to `extract_dir` + """Unpack tar/tar.gz/tar.bz2/tar.xz/tar.zst `filename` to `extract_dir` """ import tarfile # late import for breaking circular dependency try: @@ -1304,6 +1372,10 @@ def _unpack_tarfile(filename, extract_dir, *, filter=None): _UNPACK_FORMATS['xztar'] = (['.tar.xz', '.txz'], _unpack_tarfile, [], "xz'ed tar-file") +if _ZSTD_SUPPORTED: + _UNPACK_FORMATS['zstdtar'] = (['.tar.zst', '.tzst'], _unpack_tarfile, [], + "zstd'ed tar-file") + def _find_unpack_format(filename): for name, info in _UNPACK_FORMATS.items(): for extension in info[0]: @@ -1320,7 +1392,7 @@ def unpack_archive(filename, extract_dir=None, format=None, *, filter=None): is unpacked. If not provided, the current working directory is used. `format` is the archive format: one of "zip", "tar", "gztar", "bztar", - or "xztar". Or any other registered format. If not provided, + "xztar", or "zstdtar". Or any other registered format. If not provided, unpack_archive will use the filename extension and see if an unpacker was registered for that extension. @@ -1581,3 +1653,15 @@ def which(cmd, mode=os.F_OK | os.X_OK, path=None): if _access_check(name, mode): return name return None + +def __getattr__(name): + if name == "ExecError": + import warnings + warnings._deprecated( + "shutil.ExecError", + f"{warnings._DEPRECATED_MSG}; it " + "isn't raised by any shutil function.", + remove=(3, 16) + ) + return RuntimeError + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Lib/smtplib.py b/Lib/smtplib.py old mode 100755 new mode 100644 index 9b81bcfbc41..72093f7f8b0 --- a/Lib/smtplib.py +++ b/Lib/smtplib.py @@ -1,5 +1,3 @@ -#! /usr/bin/env python3 - '''SMTP/ESMTP client class. This should follow RFC 821 (SMTP), RFC 1869 (ESMTP), RFC 2554 (SMTP @@ -105,7 +103,7 @@ class SMTPSenderRefused(SMTPResponseException): """Sender address refused. In addition to the attributes set by on all SMTPResponseException - exceptions, this sets `sender' to the string that the SMTP refused. + exceptions, this sets 'sender' to the string that the SMTP refused. """ def __init__(self, code, msg, sender): @@ -182,8 +180,7 @@ def _fix_eols(data): try: hmac.digest(b'', b'', 'md5') -# except ValueError: -except (ValueError, AttributeError): # TODO: RUSTPYTHON +except ValueError: _have_cram_md5_support = False else: _have_cram_md5_support = True @@ -325,7 +322,7 @@ def _get_socket(self, host, port, timeout): def connect(self, host='localhost', port=0, source_address=None): """Connect to a host on a given port. - If the hostname ends with a colon (`:') followed by a number, and + If the hostname ends with a colon (':') followed by a number, and there is no port specified, that suffix will be stripped off and the number interpreted as the port number to use. @@ -356,7 +353,7 @@ def connect(self, host='localhost', port=0, source_address=None): return (code, msg) def send(self, s): - """Send `s' to the server.""" + """Send 's' to the server.""" if self.debuglevel > 0: self._print_debug('send:', repr(s)) if self.sock: diff --git a/Lib/socket.py b/Lib/socket.py index 35d87eff34d..727b0e75f03 100644 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -52,7 +52,9 @@ import _socket from _socket import * -import os, sys, io, selectors +import io +import os +import sys from enum import IntEnum, IntFlag try: @@ -110,102 +112,103 @@ def _intenum_converter(value, enum_klass): # WSA error codes if sys.platform.lower().startswith("win"): - errorTab = {} - errorTab[6] = "Specified event object handle is invalid." - errorTab[8] = "Insufficient memory available." - errorTab[87] = "One or more parameters are invalid." - errorTab[995] = "Overlapped operation aborted." - errorTab[996] = "Overlapped I/O event object not in signaled state." - errorTab[997] = "Overlapped operation will complete later." - errorTab[10004] = "The operation was interrupted." - errorTab[10009] = "A bad file handle was passed." - errorTab[10013] = "Permission denied." - errorTab[10014] = "A fault occurred on the network??" # WSAEFAULT - errorTab[10022] = "An invalid operation was attempted." - errorTab[10024] = "Too many open files." - errorTab[10035] = "The socket operation would block." - errorTab[10036] = "A blocking operation is already in progress." - errorTab[10037] = "Operation already in progress." - errorTab[10038] = "Socket operation on nonsocket." - errorTab[10039] = "Destination address required." - errorTab[10040] = "Message too long." - errorTab[10041] = "Protocol wrong type for socket." - errorTab[10042] = "Bad protocol option." - errorTab[10043] = "Protocol not supported." - errorTab[10044] = "Socket type not supported." - errorTab[10045] = "Operation not supported." - errorTab[10046] = "Protocol family not supported." - errorTab[10047] = "Address family not supported by protocol family." - errorTab[10048] = "The network address is in use." - errorTab[10049] = "Cannot assign requested address." - errorTab[10050] = "Network is down." - errorTab[10051] = "Network is unreachable." - errorTab[10052] = "Network dropped connection on reset." - errorTab[10053] = "Software caused connection abort." - errorTab[10054] = "The connection has been reset." - errorTab[10055] = "No buffer space available." - errorTab[10056] = "Socket is already connected." - errorTab[10057] = "Socket is not connected." - errorTab[10058] = "The network has been shut down." - errorTab[10059] = "Too many references." - errorTab[10060] = "The operation timed out." - errorTab[10061] = "Connection refused." - errorTab[10062] = "Cannot translate name." - errorTab[10063] = "The name is too long." - errorTab[10064] = "The host is down." - errorTab[10065] = "The host is unreachable." - errorTab[10066] = "Directory not empty." - errorTab[10067] = "Too many processes." - errorTab[10068] = "User quota exceeded." - errorTab[10069] = "Disk quota exceeded." - errorTab[10070] = "Stale file handle reference." - errorTab[10071] = "Item is remote." - errorTab[10091] = "Network subsystem is unavailable." - errorTab[10092] = "Winsock.dll version out of range." - errorTab[10093] = "Successful WSAStartup not yet performed." - errorTab[10101] = "Graceful shutdown in progress." - errorTab[10102] = "No more results from WSALookupServiceNext." - errorTab[10103] = "Call has been canceled." - errorTab[10104] = "Procedure call table is invalid." - errorTab[10105] = "Service provider is invalid." - errorTab[10106] = "Service provider failed to initialize." - errorTab[10107] = "System call failure." - errorTab[10108] = "Service not found." - errorTab[10109] = "Class type not found." - errorTab[10110] = "No more results from WSALookupServiceNext." - errorTab[10111] = "Call was canceled." - errorTab[10112] = "Database query was refused." - errorTab[11001] = "Host not found." - errorTab[11002] = "Nonauthoritative host not found." - errorTab[11003] = "This is a nonrecoverable error." - errorTab[11004] = "Valid name, no data record requested type." - errorTab[11005] = "QoS receivers." - errorTab[11006] = "QoS senders." - errorTab[11007] = "No QoS senders." - errorTab[11008] = "QoS no receivers." - errorTab[11009] = "QoS request confirmed." - errorTab[11010] = "QoS admission error." - errorTab[11011] = "QoS policy failure." - errorTab[11012] = "QoS bad style." - errorTab[11013] = "QoS bad object." - errorTab[11014] = "QoS traffic control error." - errorTab[11015] = "QoS generic error." - errorTab[11016] = "QoS service type error." - errorTab[11017] = "QoS flowspec error." - errorTab[11018] = "Invalid QoS provider buffer." - errorTab[11019] = "Invalid QoS filter style." - errorTab[11020] = "Invalid QoS filter style." - errorTab[11021] = "Incorrect QoS filter count." - errorTab[11022] = "Invalid QoS object length." - errorTab[11023] = "Incorrect QoS flow count." - errorTab[11024] = "Unrecognized QoS object." - errorTab[11025] = "Invalid QoS policy object." - errorTab[11026] = "Invalid QoS flow descriptor." - errorTab[11027] = "Invalid QoS provider-specific flowspec." - errorTab[11028] = "Invalid QoS provider-specific filterspec." - errorTab[11029] = "Invalid QoS shape discard mode object." - errorTab[11030] = "Invalid QoS shaping rate object." - errorTab[11031] = "Reserved policy QoS element type." + errorTab = { + 6: "Specified event object handle is invalid.", + 8: "Insufficient memory available.", + 87: "One or more parameters are invalid.", + 995: "Overlapped operation aborted.", + 996: "Overlapped I/O event object not in signaled state.", + 997: "Overlapped operation will complete later.", + 10004: "The operation was interrupted.", + 10009: "A bad file handle was passed.", + 10013: "Permission denied.", + 10014: "A fault occurred on the network??", + 10022: "An invalid operation was attempted.", + 10024: "Too many open files.", + 10035: "The socket operation would block.", + 10036: "A blocking operation is already in progress.", + 10037: "Operation already in progress.", + 10038: "Socket operation on nonsocket.", + 10039: "Destination address required.", + 10040: "Message too long.", + 10041: "Protocol wrong type for socket.", + 10042: "Bad protocol option.", + 10043: "Protocol not supported.", + 10044: "Socket type not supported.", + 10045: "Operation not supported.", + 10046: "Protocol family not supported.", + 10047: "Address family not supported by protocol family.", + 10048: "The network address is in use.", + 10049: "Cannot assign requested address.", + 10050: "Network is down.", + 10051: "Network is unreachable.", + 10052: "Network dropped connection on reset.", + 10053: "Software caused connection abort.", + 10054: "The connection has been reset.", + 10055: "No buffer space available.", + 10056: "Socket is already connected.", + 10057: "Socket is not connected.", + 10058: "The network has been shut down.", + 10059: "Too many references.", + 10060: "The operation timed out.", + 10061: "Connection refused.", + 10062: "Cannot translate name.", + 10063: "The name is too long.", + 10064: "The host is down.", + 10065: "The host is unreachable.", + 10066: "Directory not empty.", + 10067: "Too many processes.", + 10068: "User quota exceeded.", + 10069: "Disk quota exceeded.", + 10070: "Stale file handle reference.", + 10071: "Item is remote.", + 10091: "Network subsystem is unavailable.", + 10092: "Winsock.dll version out of range.", + 10093: "Successful WSAStartup not yet performed.", + 10101: "Graceful shutdown in progress.", + 10102: "No more results from WSALookupServiceNext.", + 10103: "Call has been canceled.", + 10104: "Procedure call table is invalid.", + 10105: "Service provider is invalid.", + 10106: "Service provider failed to initialize.", + 10107: "System call failure.", + 10108: "Service not found.", + 10109: "Class type not found.", + 10110: "No more results from WSALookupServiceNext.", + 10111: "Call was canceled.", + 10112: "Database query was refused.", + 11001: "Host not found.", + 11002: "Nonauthoritative host not found.", + 11003: "This is a nonrecoverable error.", + 11004: "Valid name, no data record requested type.", + 11005: "QoS receivers.", + 11006: "QoS senders.", + 11007: "No QoS senders.", + 11008: "QoS no receivers.", + 11009: "QoS request confirmed.", + 11010: "QoS admission error.", + 11011: "QoS policy failure.", + 11012: "QoS bad style.", + 11013: "QoS bad object.", + 11014: "QoS traffic control error.", + 11015: "QoS generic error.", + 11016: "QoS service type error.", + 11017: "QoS flowspec error.", + 11018: "Invalid QoS provider buffer.", + 11019: "Invalid QoS filter style.", + 11020: "Invalid QoS filter style.", + 11021: "Incorrect QoS filter count.", + 11022: "Invalid QoS object length.", + 11023: "Incorrect QoS flow count.", + 11024: "Unrecognized QoS object.", + 11025: "Invalid QoS policy object.", + 11026: "Invalid QoS flow descriptor.", + 11027: "Invalid QoS provider-specific flowspec.", + 11028: "Invalid QoS provider-specific filterspec.", + 11029: "Invalid QoS shape discard mode object.", + 11030: "Invalid QoS shaping rate object.", + 11031: "Reserved policy QoS element type." + } __all__.append("errorTab") @@ -348,6 +351,9 @@ def makefile(self, mode="r", buffering=None, *, if hasattr(os, 'sendfile'): def _sendfile_use_sendfile(self, file, offset=0, count=None): + # Lazy import to improve module import time + import selectors + self._check_sendfile_params(file, offset, count) sockno = self.fileno() try: @@ -549,20 +555,18 @@ def fromfd(fd, family, type, proto=0): return socket(family, type, proto, nfd) if hasattr(_socket.socket, "sendmsg"): - import array - def send_fds(sock, buffers, fds, flags=0, address=None): """ send_fds(sock, buffers, fds[, flags[, address]]) -> integer Send the list of file descriptors fds over an AF_UNIX socket. """ + import array + return sock.sendmsg(buffers, [(_socket.SOL_SOCKET, _socket.SCM_RIGHTS, array.array("i", fds))]) __all__.append("send_fds") if hasattr(_socket.socket, "recvmsg"): - import array - def recv_fds(sock, bufsize, maxfds, flags=0): """ recv_fds(sock, bufsize, maxfds[, flags]) -> (data, list of file descriptors, msg_flags, address) @@ -570,6 +574,8 @@ def recv_fds(sock, bufsize, maxfds, flags=0): Receive up to maxfds file descriptors returning the message data and a list containing the descriptors. """ + import array + # Array of ints fds = array.array("i") msg, ancdata, flags, addr = sock.recvmsg(bufsize, diff --git a/Lib/ssl.py b/Lib/ssl.py index c8703b046cf..8889aff92fa 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -116,7 +116,7 @@ from _ssl import ( HAS_SNI, HAS_ECDH, HAS_NPN, HAS_ALPN, HAS_SSLv2, HAS_SSLv3, HAS_TLSv1, - HAS_TLSv1_1, HAS_TLSv1_2, HAS_TLSv1_3, HAS_PSK + HAS_TLSv1_1, HAS_TLSv1_2, HAS_TLSv1_3, HAS_PSK, HAS_PHA ) from _ssl import _DEFAULT_CIPHERS, _OPENSSL_API_VERSION @@ -186,7 +186,7 @@ class _TLSContentType: class _TLSAlertType: """Alert types for TLSContentType.ALERT messages - See RFC 8466, section B.2 + See RFC 8446, section B.2 """ CLOSE_NOTIFY = 0 UNEXPECTED_MESSAGE = 10 diff --git a/Lib/stat.py b/Lib/stat.py index 1b4ed1ebc94..81f694329bf 100644 --- a/Lib/stat.py +++ b/Lib/stat.py @@ -166,9 +166,14 @@ def filemode(mode): perm = [] for index, table in enumerate(_filemode_table): for bit, char in table: - if mode & bit == bit: - perm.append(char) - break + if index == 0: + if S_IFMT(mode) == bit: + perm.append(char) + break + else: + if mode & bit == bit: + perm.append(char) + break else: if index == 0: # Unknown filetype diff --git a/Lib/statistics.py b/Lib/statistics.py index ad4a94219cf..26cf925529e 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -138,7 +138,7 @@ from decimal import Decimal from itertools import count, groupby, repeat from bisect import bisect_left, bisect_right -from math import hypot, sqrt, fabs, exp, erf, tau, log, fsum, sumprod +from math import hypot, sqrt, fabs, exp, erfc, tau, log, fsum, sumprod from math import isfinite, isinf, pi, cos, sin, tan, cosh, asin, atan, acos from functools import reduce from operator import itemgetter @@ -147,447 +147,149 @@ _SQRT2 = sqrt(2.0) _random = random -# === Exceptions === +## Exceptions ############################################################## class StatisticsError(ValueError): pass -# === Private utilities === +## Measures of central tendency (averages) ################################# -def _sum(data): - """_sum(data) -> (type, sum, count) - - Return a high-precision sum of the given numeric data as a fraction, - together with the type to be converted to and the count of items. - - Examples - -------- - - >>> _sum([3, 2.25, 4.5, -0.5, 0.25]) - (, Fraction(19, 2), 5) - - Some sources of round-off error will be avoided: - - # Built-in sum returns zero. - >>> _sum([1e50, 1, -1e50] * 1000) - (, Fraction(1000, 1), 3000) +def mean(data): + """Return the sample arithmetic mean of data. - Fractions and Decimals are also supported: + >>> mean([1, 2, 3, 4, 4]) + 2.8 >>> from fractions import Fraction as F - >>> _sum([F(2, 3), F(7, 5), F(1, 4), F(5, 6)]) - (, Fraction(63, 20), 4) + >>> mean([F(3, 7), F(1, 21), F(5, 3), F(1, 3)]) + Fraction(13, 21) >>> from decimal import Decimal as D - >>> data = [D("0.1375"), D("0.2108"), D("0.3061"), D("0.0419")] - >>> _sum(data) - (, Fraction(6963, 10000), 4) + >>> mean([D("0.5"), D("0.75"), D("0.625"), D("0.375")]) + Decimal('0.5625') + + If ``data`` is empty, StatisticsError will be raised. - Mixed types are currently treated as an error, except that int is - allowed. """ - count = 0 - types = set() - types_add = types.add - partials = {} - partials_get = partials.get - for typ, values in groupby(data, type): - types_add(typ) - for n, d in map(_exact_ratio, values): - count += 1 - partials[d] = partials_get(d, 0) + n - if None in partials: - # The sum will be a NAN or INF. We can ignore all the finite - # partials, and just look at this special one. - total = partials[None] - assert not _isfinite(total) - else: - # Sum all the partial sums using builtin sum. - total = sum(Fraction(n, d) for d, n in partials.items()) - T = reduce(_coerce, types, int) # or raise TypeError - return (T, total, count) + T, total, n = _sum(data) + if n < 1: + raise StatisticsError('mean requires at least one data point') + return _convert(total / n, T) -def _ss(data, c=None): - """Return the exact mean and sum of square deviations of sequence data. +def fmean(data, weights=None): + """Convert data to floats and compute the arithmetic mean. - Calculations are done in a single pass, allowing the input to be an iterator. + This runs faster than the mean() function and it always returns a float. + If the input dataset is empty, it raises a StatisticsError. - If given *c* is used the mean; otherwise, it is calculated from the data. - Use the *c* argument with care, as it can lead to garbage results. + >>> fmean([3.5, 4.0, 5.25]) + 4.25 """ - if c is not None: - T, ssd, count = _sum((d := x - c) * d for x in data) - return (T, ssd, c, count) - count = 0 - types = set() - types_add = types.add - sx_partials = defaultdict(int) - sxx_partials = defaultdict(int) - for typ, values in groupby(data, type): - types_add(typ) - for n, d in map(_exact_ratio, values): - count += 1 - sx_partials[d] += n - sxx_partials[d] += n * n - if not count: - ssd = c = Fraction(0) - elif None in sx_partials: - # The sum will be a NAN or INF. We can ignore all the finite - # partials, and just look at this special one. - ssd = c = sx_partials[None] - assert not _isfinite(ssd) - else: - sx = sum(Fraction(n, d) for d, n in sx_partials.items()) - sxx = sum(Fraction(n, d*d) for d, n in sxx_partials.items()) - # This formula has poor numeric properties for floats, - # but with fractions it is exact. - ssd = (count * sxx - sx * sx) / count - c = sx / count - T = reduce(_coerce, types, int) # or raise TypeError - return (T, ssd, c, count) + if weights is None: + try: + n = len(data) + except TypeError: + # Handle iterators that do not define __len__(). + counter = count() + total = fsum(map(itemgetter(0), zip(data, counter))) + n = next(counter) + else: + total = fsum(data) -def _isfinite(x): - try: - return x.is_finite() # Likely a Decimal. - except AttributeError: - return math.isfinite(x) # Coerces to float first. + if not n: + raise StatisticsError('fmean requires at least one data point') + return total / n -def _coerce(T, S): - """Coerce types T and S to a common type, or raise TypeError. + if not isinstance(weights, (list, tuple)): + weights = list(weights) - Coercion rules are currently an implementation detail. See the CoerceTest - test class in test_statistics for details. - """ - # See http://bugs.python.org/issue24068. - assert T is not bool, "initial type T is bool" - # If the types are the same, no need to coerce anything. Put this - # first, so that the usual case (no coercion needed) happens as soon - # as possible. - if T is S: return T - # Mixed int & other coerce to the other type. - if S is int or S is bool: return T - if T is int: return S - # If one is a (strict) subclass of the other, coerce to the subclass. - if issubclass(S, T): return S - if issubclass(T, S): return T - # Ints coerce to the other type. - if issubclass(T, int): return S - if issubclass(S, int): return T - # Mixed fraction & float coerces to float (or float subclass). - if issubclass(T, Fraction) and issubclass(S, float): - return S - if issubclass(T, float) and issubclass(S, Fraction): - return T - # Any other combination is disallowed. - msg = "don't know how to coerce %s and %s" - raise TypeError(msg % (T.__name__, S.__name__)) + try: + num = sumprod(data, weights) + except ValueError: + raise StatisticsError('data and weights must be the same length') + den = fsum(weights) -def _exact_ratio(x): - """Return Real number x to exact (numerator, denominator) pair. + if not den: + raise StatisticsError('sum of weights must be non-zero') - >>> _exact_ratio(0.25) - (1, 4) + return num / den - x is expected to be an int, Fraction, Decimal or float. - """ - # XXX We should revisit whether using fractions to accumulate exact - # ratios is the right way to go. +def geometric_mean(data): + """Convert data to floats and compute the geometric mean. - # The integer ratios for binary floats can have numerators or - # denominators with over 300 decimal digits. The problem is more - # acute with decimal floats where the default decimal context - # supports a huge range of exponents from Emin=-999999 to - # Emax=999999. When expanded with as_integer_ratio(), numbers like - # Decimal('3.14E+5000') and Decimal('3.14E-5000') have large - # numerators or denominators that will slow computation. + Raises a StatisticsError if the input dataset is empty + or if it contains a negative value. - # When the integer ratios are accumulated as fractions, the size - # grows to cover the full range from the smallest magnitude to the - # largest. For example, Fraction(3.14E+300) + Fraction(3.14E-300), - # has a 616 digit numerator. Likewise, - # Fraction(Decimal('3.14E+5000')) + Fraction(Decimal('3.14E-5000')) - # has 10,003 digit numerator. + Returns zero if the product of inputs is zero. - # This doesn't seem to have been problem in practice, but it is a - # potential pitfall. + No special efforts are made to achieve exact results. + (However, this may change in the future.) - try: - return x.as_integer_ratio() - except AttributeError: - pass - except (OverflowError, ValueError): - # float NAN or INF. - assert not _isfinite(x) - return (x, None) - try: - # x may be an Integral ABC. - return (x.numerator, x.denominator) - except AttributeError: - msg = f"can't convert type '{type(x).__name__}' to numerator/denominator" - raise TypeError(msg) + >>> round(geometric_mean([54, 24, 36]), 9) + 36.0 + """ + n = 0 + found_zero = False -def _convert(value, T): - """Convert value to given numeric type T.""" - if type(value) is T: - # This covers the cases where T is Fraction, or where value is - # a NAN or INF (Decimal or float). - return value - if issubclass(T, int) and value.denominator != 1: - T = float - try: - # FIXME: what do we do if this overflows? - return T(value) - except TypeError: - if issubclass(T, Decimal): - return T(value.numerator) / T(value.denominator) - else: - raise + def count_positive(iterable): + nonlocal n, found_zero + for n, x in enumerate(iterable, start=1): + if x > 0.0 or math.isnan(x): + yield x + elif x == 0.0: + found_zero = True + else: + raise StatisticsError('No negative inputs allowed', x) + total = fsum(map(log, count_positive(data))) -def _fail_neg(values, errmsg='negative value'): - """Iterate over values, failing if any are less than zero.""" - for x in values: - if x < 0: - raise StatisticsError(errmsg) - yield x + if not n: + raise StatisticsError('Must have a non-empty dataset') + if math.isnan(total): + return math.nan + if found_zero: + return math.nan if total == math.inf else 0.0 + return exp(total / n) -def _rank(data, /, *, key=None, reverse=False, ties='average', start=1) -> list[float]: - """Rank order a dataset. The lowest value has rank 1. - Ties are averaged so that equal values receive the same rank: +def harmonic_mean(data, weights=None): + """Return the harmonic mean of data. - >>> data = [31, 56, 31, 25, 75, 18] - >>> _rank(data) - [3.5, 5.0, 3.5, 2.0, 6.0, 1.0] + The harmonic mean is the reciprocal of the arithmetic mean of the + reciprocals of the data. It can be used for averaging ratios or + rates, for example speeds. - The operation is idempotent: + Suppose a car travels 40 km/hr for 5 km and then speeds-up to + 60 km/hr for another 5 km. What is the average speed? - >>> _rank([3.5, 5.0, 3.5, 2.0, 6.0, 1.0]) - [3.5, 5.0, 3.5, 2.0, 6.0, 1.0] + >>> harmonic_mean([40, 60]) + 48.0 - It is possible to rank the data in reverse order so that the - highest value has rank 1. Also, a key-function can extract - the field to be ranked: + Suppose a car travels 40 km/hr for 5 km, and when traffic clears, + speeds-up to 60 km/hr for the remaining 30 km of the journey. What + is the average speed? - >>> goals = [('eagles', 45), ('bears', 48), ('lions', 44)] - >>> _rank(goals, key=itemgetter(1), reverse=True) - [2.0, 1.0, 3.0] + >>> harmonic_mean([40, 60], weights=[5, 30]) + 56.0 - Ranks are conventionally numbered starting from one; however, - setting *start* to zero allows the ranks to be used as array indices: + If ``data`` is empty, or any element is less than zero, + ``harmonic_mean`` will raise ``StatisticsError``. - >>> prize = ['Gold', 'Silver', 'Bronze', 'Certificate'] - >>> scores = [8.1, 7.3, 9.4, 8.3] - >>> [prize[int(i)] for i in _rank(scores, start=0, reverse=True)] - ['Bronze', 'Certificate', 'Gold', 'Silver'] - - """ - # If this function becomes public at some point, more thought - # needs to be given to the signature. A list of ints is - # plausible when ties is "min" or "max". When ties is "average", - # either list[float] or list[Fraction] is plausible. - - # Default handling of ties matches scipy.stats.mstats.spearmanr. - if ties != 'average': - raise ValueError(f'Unknown tie resolution method: {ties!r}') - if key is not None: - data = map(key, data) - val_pos = sorted(zip(data, count()), reverse=reverse) - i = start - 1 - result = [0] * len(val_pos) - for _, g in groupby(val_pos, key=itemgetter(0)): - group = list(g) - size = len(group) - rank = i + (size + 1) / 2 - for value, orig_pos in group: - result[orig_pos] = rank - i += size - return result - - -def _integer_sqrt_of_frac_rto(n: int, m: int) -> int: - """Square root of n/m, rounded to the nearest integer using round-to-odd.""" - # Reference: https://www.lri.fr/~melquion/doc/05-imacs17_1-expose.pdf - a = math.isqrt(n // m) - return a | (a*a*m != n) - - -# For 53 bit precision floats, the bit width used in -# _float_sqrt_of_frac() is 109. -_sqrt_bit_width: int = 2 * sys.float_info.mant_dig + 3 - - -def _float_sqrt_of_frac(n: int, m: int) -> float: - """Square root of n/m as a float, correctly rounded.""" - # See principle and proof sketch at: https://bugs.python.org/msg407078 - q = (n.bit_length() - m.bit_length() - _sqrt_bit_width) // 2 - if q >= 0: - numerator = _integer_sqrt_of_frac_rto(n, m << 2 * q) << q - denominator = 1 - else: - numerator = _integer_sqrt_of_frac_rto(n << -2 * q, m) - denominator = 1 << -q - return numerator / denominator # Convert to float - - -def _decimal_sqrt_of_frac(n: int, m: int) -> Decimal: - """Square root of n/m as a Decimal, correctly rounded.""" - # Premise: For decimal, computing (n/m).sqrt() can be off - # by 1 ulp from the correctly rounded result. - # Method: Check the result, moving up or down a step if needed. - if n <= 0: - if not n: - return Decimal('0.0') - n, m = -n, -m - - root = (Decimal(n) / Decimal(m)).sqrt() - nr, dr = root.as_integer_ratio() - - plus = root.next_plus() - np, dp = plus.as_integer_ratio() - # test: n / m > ((root + plus) / 2) ** 2 - if 4 * n * (dr*dp)**2 > m * (dr*np + dp*nr)**2: - return plus - - minus = root.next_minus() - nm, dm = minus.as_integer_ratio() - # test: n / m < ((root + minus) / 2) ** 2 - if 4 * n * (dr*dm)**2 < m * (dr*nm + dm*nr)**2: - return minus - - return root - - -# === Measures of central tendency (averages) === - -def mean(data): - """Return the sample arithmetic mean of data. - - >>> mean([1, 2, 3, 4, 4]) - 2.8 - - >>> from fractions import Fraction as F - >>> mean([F(3, 7), F(1, 21), F(5, 3), F(1, 3)]) - Fraction(13, 21) - - >>> from decimal import Decimal as D - >>> mean([D("0.5"), D("0.75"), D("0.625"), D("0.375")]) - Decimal('0.5625') - - If ``data`` is empty, StatisticsError will be raised. - """ - T, total, n = _sum(data) - if n < 1: - raise StatisticsError('mean requires at least one data point') - return _convert(total / n, T) - - -def fmean(data, weights=None): - """Convert data to floats and compute the arithmetic mean. - - This runs faster than the mean() function and it always returns a float. - If the input dataset is empty, it raises a StatisticsError. - - >>> fmean([3.5, 4.0, 5.25]) - 4.25 - """ - if weights is None: - try: - n = len(data) - except TypeError: - # Handle iterators that do not define __len__(). - n = 0 - def count(iterable): - nonlocal n - for n, x in enumerate(iterable, start=1): - yield x - data = count(data) - total = fsum(data) - if not n: - raise StatisticsError('fmean requires at least one data point') - return total / n - if not isinstance(weights, (list, tuple)): - weights = list(weights) - try: - num = sumprod(data, weights) - except ValueError: - raise StatisticsError('data and weights must be the same length') - den = fsum(weights) - if not den: - raise StatisticsError('sum of weights must be non-zero') - return num / den - - -def geometric_mean(data): - """Convert data to floats and compute the geometric mean. - - Raises a StatisticsError if the input dataset is empty - or if it contains a negative value. - - Returns zero if the product of inputs is zero. - - No special efforts are made to achieve exact results. - (However, this may change in the future.) - - >>> round(geometric_mean([54, 24, 36]), 9) - 36.0 - """ - n = 0 - found_zero = False - def count_positive(iterable): - nonlocal n, found_zero - for n, x in enumerate(iterable, start=1): - if x > 0.0 or math.isnan(x): - yield x - elif x == 0.0: - found_zero = True - else: - raise StatisticsError('No negative inputs allowed', x) - total = fsum(map(log, count_positive(data))) - if not n: - raise StatisticsError('Must have a non-empty dataset') - if math.isnan(total): - return math.nan - if found_zero: - return math.nan if total == math.inf else 0.0 - return exp(total / n) - - -def harmonic_mean(data, weights=None): - """Return the harmonic mean of data. - - The harmonic mean is the reciprocal of the arithmetic mean of the - reciprocals of the data. It can be used for averaging ratios or - rates, for example speeds. - - Suppose a car travels 40 km/hr for 5 km and then speeds-up to - 60 km/hr for another 5 km. What is the average speed? - - >>> harmonic_mean([40, 60]) - 48.0 - - Suppose a car travels 40 km/hr for 5 km, and when traffic clears, - speeds-up to 60 km/hr for the remaining 30 km of the journey. What - is the average speed? - - >>> harmonic_mean([40, 60], weights=[5, 30]) - 56.0 - - If ``data`` is empty, or any element is less than zero, - ``harmonic_mean`` will raise ``StatisticsError``. """ if iter(data) is data: data = list(data) + errmsg = 'harmonic mean does not support negative values' + n = len(data) if n < 1: raise StatisticsError('harmonic_mean requires at least one data point') @@ -599,6 +301,7 @@ def harmonic_mean(data, weights=None): return x else: raise TypeError('unsupported type') + if weights is None: weights = repeat(1, n) sum_weights = n @@ -608,16 +311,19 @@ def harmonic_mean(data, weights=None): if len(weights) != n: raise StatisticsError('Number of weights does not match data size') _, sum_weights, _ = _sum(w for w in _fail_neg(weights, errmsg)) + try: data = _fail_neg(data, errmsg) T, total, count = _sum(w / x if w else 0 for w, x in zip(weights, data)) except ZeroDivisionError: return 0 + if total <= 0: raise StatisticsError('Weighted sum must be positive') + return _convert(sum_weights / total, T) -# FIXME: investigate ways to calculate medians without sorting? Quickselect? + def median(data): """Return the median (middle value) of numeric data. @@ -654,6 +360,9 @@ def median_low(data): 3 """ + # Potentially the sorting step could be replaced with a quickselect. + # However, it would require an excellent implementation to beat our + # highly optimized builtin sort. data = sorted(data) n = len(data) if n == 0: @@ -797,6 +506,7 @@ def multimode(data): ['b', 'd', 'f'] >>> multimode('') [] + """ counts = Counter(iter(data)) if not counts: @@ -805,347 +515,48 @@ def multimode(data): return [value for value, count in counts.items() if count == maxcount] -def kde(data, h, kernel='normal', *, cumulative=False): - """Kernel Density Estimation: Create a continuous probability density - function or cumulative distribution function from discrete samples. +## Measures of spread ###################################################### - The basic idea is to smooth the data using a kernel function - to help draw inferences about a population from a sample. +def variance(data, xbar=None): + """Return the sample variance of data. - The degree of smoothing is controlled by the scaling parameter h - which is called the bandwidth. Smaller values emphasize local - features while larger values give smoother results. + data should be an iterable of Real-valued numbers, with at least two + values. The optional argument xbar, if given, should be the mean of + the data. If it is missing or None, the mean is automatically calculated. - The kernel determines the relative weights of the sample data - points. Generally, the choice of kernel shape does not matter - as much as the more influential bandwidth smoothing parameter. + Use this function when your data is a sample from a population. To + calculate the variance from the entire population, see ``pvariance``. - Kernels that give some weight to every sample point: + Examples: - normal (gauss) - logistic - sigmoid + >>> data = [2.75, 1.75, 1.25, 0.25, 0.5, 1.25, 3.5] + >>> variance(data) + 1.3720238095238095 - Kernels that only give weight to sample points within - the bandwidth: + If you have already calculated the mean of your data, you can pass it as + the optional second argument ``xbar`` to avoid recalculating it: - rectangular (uniform) - triangular - parabolic (epanechnikov) - quartic (biweight) - triweight - cosine + >>> m = mean(data) + >>> variance(data, m) + 1.3720238095238095 - If *cumulative* is true, will return a cumulative distribution function. + This function does not check that ``xbar`` is actually the mean of + ``data``. Giving arbitrary values for ``xbar`` may lead to invalid or + impossible results. - A StatisticsError will be raised if the data sequence is empty. + Decimals and Fractions are supported: - Example - ------- + >>> from decimal import Decimal as D + >>> variance([D("27.5"), D("30.25"), D("30.25"), D("34.5"), D("41.75")]) + Decimal('31.01875') - Given a sample of six data points, construct a continuous - function that estimates the underlying probability density: + >>> from fractions import Fraction as F + >>> variance([F(1, 6), F(1, 2), F(5, 3)]) + Fraction(67, 108) - >>> sample = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] - >>> f_hat = kde(sample, h=1.5) - - Compute the area under the curve: - - >>> area = sum(f_hat(x) for x in range(-20, 20)) - >>> round(area, 4) - 1.0 - - Plot the estimated probability density function at - evenly spaced points from -6 to 10: - - >>> for x in range(-6, 11): - ... density = f_hat(x) - ... plot = ' ' * int(density * 400) + 'x' - ... print(f'{x:2}: {density:.3f} {plot}') - ... - -6: 0.002 x - -5: 0.009 x - -4: 0.031 x - -3: 0.070 x - -2: 0.111 x - -1: 0.125 x - 0: 0.110 x - 1: 0.086 x - 2: 0.068 x - 3: 0.059 x - 4: 0.066 x - 5: 0.082 x - 6: 0.082 x - 7: 0.058 x - 8: 0.028 x - 9: 0.009 x - 10: 0.002 x - - Estimate P(4.5 < X <= 7.5), the probability that a new sample value - will be between 4.5 and 7.5: - - >>> cdf = kde(sample, h=1.5, cumulative=True) - >>> round(cdf(7.5) - cdf(4.5), 2) - 0.22 - - References - ---------- - - Kernel density estimation and its application: - https://www.itm-conferences.org/articles/itmconf/pdf/2018/08/itmconf_sam2018_00037.pdf - - Kernel functions in common use: - https://en.wikipedia.org/wiki/Kernel_(statistics)#kernel_functions_in_common_use - - Interactive graphical demonstration and exploration: - https://demonstrations.wolfram.com/KernelDensityEstimation/ - - Kernel estimation of cumulative distribution function of a random variable with bounded support - https://www.econstor.eu/bitstream/10419/207829/1/10.21307_stattrans-2016-037.pdf - - """ - - n = len(data) - if not n: - raise StatisticsError('Empty data sequence') - - if not isinstance(data[0], (int, float)): - raise TypeError('Data sequence must contain ints or floats') - - if h <= 0.0: - raise StatisticsError(f'Bandwidth h must be positive, not {h=!r}') - - match kernel: - - case 'normal' | 'gauss': - sqrt2pi = sqrt(2 * pi) - sqrt2 = sqrt(2) - K = lambda t: exp(-1/2 * t * t) / sqrt2pi - W = lambda t: 1/2 * (1.0 + erf(t / sqrt2)) - support = None - - case 'logistic': - # 1.0 / (exp(t) + 2.0 + exp(-t)) - K = lambda t: 1/2 / (1.0 + cosh(t)) - W = lambda t: 1.0 - 1.0 / (exp(t) + 1.0) - support = None - - case 'sigmoid': - # (2/pi) / (exp(t) + exp(-t)) - c1 = 1 / pi - c2 = 2 / pi - K = lambda t: c1 / cosh(t) - W = lambda t: c2 * atan(exp(t)) - support = None - - case 'rectangular' | 'uniform': - K = lambda t: 1/2 - W = lambda t: 1/2 * t + 1/2 - support = 1.0 - - case 'triangular': - K = lambda t: 1.0 - abs(t) - W = lambda t: t*t * (1/2 if t < 0.0 else -1/2) + t + 1/2 - support = 1.0 - - case 'parabolic' | 'epanechnikov': - K = lambda t: 3/4 * (1.0 - t * t) - W = lambda t: -1/4 * t**3 + 3/4 * t + 1/2 - support = 1.0 - - case 'quartic' | 'biweight': - K = lambda t: 15/16 * (1.0 - t * t) ** 2 - W = lambda t: 3/16 * t**5 - 5/8 * t**3 + 15/16 * t + 1/2 - support = 1.0 - - case 'triweight': - K = lambda t: 35/32 * (1.0 - t * t) ** 3 - W = lambda t: 35/32 * (-1/7*t**7 + 3/5*t**5 - t**3 + t) + 1/2 - support = 1.0 - - case 'cosine': - c1 = pi / 4 - c2 = pi / 2 - K = lambda t: c1 * cos(c2 * t) - W = lambda t: 1/2 * sin(c2 * t) + 1/2 - support = 1.0 - - case _: - raise StatisticsError(f'Unknown kernel name: {kernel!r}') - - if support is None: - - def pdf(x): - n = len(data) - return sum(K((x - x_i) / h) for x_i in data) / (n * h) - - def cdf(x): - n = len(data) - return sum(W((x - x_i) / h) for x_i in data) / n - - else: - - sample = sorted(data) - bandwidth = h * support - - def pdf(x): - nonlocal n, sample - if len(data) != n: - sample = sorted(data) - n = len(data) - i = bisect_left(sample, x - bandwidth) - j = bisect_right(sample, x + bandwidth) - supported = sample[i : j] - return sum(K((x - x_i) / h) for x_i in supported) / (n * h) - - def cdf(x): - nonlocal n, sample - if len(data) != n: - sample = sorted(data) - n = len(data) - i = bisect_left(sample, x - bandwidth) - j = bisect_right(sample, x + bandwidth) - supported = sample[i : j] - return sum((W((x - x_i) / h) for x_i in supported), i) / n - - if cumulative: - cdf.__doc__ = f'CDF estimate with {h=!r} and {kernel=!r}' - return cdf - - else: - pdf.__doc__ = f'PDF estimate with {h=!r} and {kernel=!r}' - return pdf - - -# Notes on methods for computing quantiles -# ---------------------------------------- -# -# There is no one perfect way to compute quantiles. Here we offer -# two methods that serve common needs. Most other packages -# surveyed offered at least one or both of these two, making them -# "standard" in the sense of "widely-adopted and reproducible". -# They are also easy to explain, easy to compute manually, and have -# straight-forward interpretations that aren't surprising. - -# The default method is known as "R6", "PERCENTILE.EXC", or "expected -# value of rank order statistics". The alternative method is known as -# "R7", "PERCENTILE.INC", or "mode of rank order statistics". - -# For sample data where there is a positive probability for values -# beyond the range of the data, the R6 exclusive method is a -# reasonable choice. Consider a random sample of nine values from a -# population with a uniform distribution from 0.0 to 1.0. The -# distribution of the third ranked sample point is described by -# betavariate(alpha=3, beta=7) which has mode=0.250, median=0.286, and -# mean=0.300. Only the latter (which corresponds with R6) gives the -# desired cut point with 30% of the population falling below that -# value, making it comparable to a result from an inv_cdf() function. -# The R6 exclusive method is also idempotent. - -# For describing population data where the end points are known to -# be included in the data, the R7 inclusive method is a reasonable -# choice. Instead of the mean, it uses the mode of the beta -# distribution for the interior points. Per Hyndman & Fan, "One nice -# property is that the vertices of Q7(p) divide the range into n - 1 -# intervals, and exactly 100p% of the intervals lie to the left of -# Q7(p) and 100(1 - p)% of the intervals lie to the right of Q7(p)." - -# If needed, other methods could be added. However, for now, the -# position is that fewer options make for easier choices and that -# external packages can be used for anything more advanced. - -def quantiles(data, *, n=4, method='exclusive'): - """Divide *data* into *n* continuous intervals with equal probability. - - Returns a list of (n - 1) cut points separating the intervals. - - Set *n* to 4 for quartiles (the default). Set *n* to 10 for deciles. - Set *n* to 100 for percentiles which gives the 99 cuts points that - separate *data* in to 100 equal sized groups. - - The *data* can be any iterable containing sample. - The cut points are linearly interpolated between data points. - - If *method* is set to *inclusive*, *data* is treated as population - data. The minimum value is treated as the 0th percentile and the - maximum value is treated as the 100th percentile. """ - if n < 1: - raise StatisticsError('n must be at least 1') - data = sorted(data) - ld = len(data) - if ld < 2: - if ld == 1: - return data * (n - 1) - raise StatisticsError('must have at least one data point') - - if method == 'inclusive': - m = ld - 1 - result = [] - for i in range(1, n): - j, delta = divmod(i * m, n) - interpolated = (data[j] * (n - delta) + data[j + 1] * delta) / n - result.append(interpolated) - return result - - if method == 'exclusive': - m = ld + 1 - result = [] - for i in range(1, n): - j = i * m // n # rescale i to m/n - j = 1 if j < 1 else ld-1 if j > ld-1 else j # clamp to 1 .. ld-1 - delta = i*m - j*n # exact integer math - interpolated = (data[j - 1] * (n - delta) + data[j] * delta) / n - result.append(interpolated) - return result - - raise ValueError(f'Unknown method: {method!r}') - - -# === Measures of spread === - -# See http://mathworld.wolfram.com/Variance.html -# http://mathworld.wolfram.com/SampleVariance.html + # http://mathworld.wolfram.com/SampleVariance.html - -def variance(data, xbar=None): - """Return the sample variance of data. - - data should be an iterable of Real-valued numbers, with at least two - values. The optional argument xbar, if given, should be the mean of - the data. If it is missing or None, the mean is automatically calculated. - - Use this function when your data is a sample from a population. To - calculate the variance from the entire population, see ``pvariance``. - - Examples: - - >>> data = [2.75, 1.75, 1.25, 0.25, 0.5, 1.25, 3.5] - >>> variance(data) - 1.3720238095238095 - - If you have already calculated the mean of your data, you can pass it as - the optional second argument ``xbar`` to avoid recalculating it: - - >>> m = mean(data) - >>> variance(data, m) - 1.3720238095238095 - - This function does not check that ``xbar`` is actually the mean of - ``data``. Giving arbitrary values for ``xbar`` may lead to invalid or - impossible results. - - Decimals and Fractions are supported: - - >>> from decimal import Decimal as D - >>> variance([D("27.5"), D("30.25"), D("30.25"), D("34.5"), D("41.75")]) - Decimal('31.01875') - - >>> from fractions import Fraction as F - >>> variance([F(1, 6), F(1, 2), F(5, 3)]) - Fraction(67, 108) - - """ T, ss, c, n = _ss(data, xbar) if n < 2: raise StatisticsError('variance requires at least two data points') @@ -1187,6 +598,8 @@ def pvariance(data, mu=None): Fraction(13, 72) """ + # http://mathworld.wolfram.com/Variance.html + T, ss, c, n = _ss(data, mu) if n < 1: raise StatisticsError('pvariance requires at least one data point') @@ -1206,9 +619,14 @@ def stdev(data, xbar=None): if n < 2: raise StatisticsError('stdev requires at least two data points') mss = ss / (n - 1) + try: + mss_numerator = mss.numerator + mss_denominator = mss.denominator + except AttributeError: + raise ValueError('inf or nan encountered in data') if issubclass(T, Decimal): - return _decimal_sqrt_of_frac(mss.numerator, mss.denominator) - return _float_sqrt_of_frac(mss.numerator, mss.denominator) + return _decimal_sqrt_of_frac(mss_numerator, mss_denominator) + return _float_sqrt_of_frac(mss_numerator, mss_denominator) def pstdev(data, mu=None): @@ -1224,51 +642,17 @@ def pstdev(data, mu=None): if n < 1: raise StatisticsError('pstdev requires at least one data point') mss = ss / n + try: + mss_numerator = mss.numerator + mss_denominator = mss.denominator + except AttributeError: + raise ValueError('inf or nan encountered in data') if issubclass(T, Decimal): - return _decimal_sqrt_of_frac(mss.numerator, mss.denominator) - return _float_sqrt_of_frac(mss.numerator, mss.denominator) + return _decimal_sqrt_of_frac(mss_numerator, mss_denominator) + return _float_sqrt_of_frac(mss_numerator, mss_denominator) -def _mean_stdev(data): - """In one pass, compute the mean and sample standard deviation as floats.""" - T, ss, xbar, n = _ss(data) - if n < 2: - raise StatisticsError('stdev requires at least two data points') - mss = ss / (n - 1) - try: - return float(xbar), _float_sqrt_of_frac(mss.numerator, mss.denominator) - except AttributeError: - # Handle Nans and Infs gracefully - return float(xbar), float(xbar) / float(ss) - -def _sqrtprod(x: float, y: float) -> float: - "Return sqrt(x * y) computed with improved accuracy and without overflow/underflow." - h = sqrt(x * y) - if not isfinite(h): - if isinf(h) and not isinf(x) and not isinf(y): - # Finite inputs overflowed, so scale down, and recompute. - scale = 2.0 ** -512 # sqrt(1 / sys.float_info.max) - return _sqrtprod(scale * x, scale * y) / scale - return h - if not h: - if x and y: - # Non-zero inputs underflowed, so scale up, and recompute. - # Scale: 1 / sqrt(sys.float_info.min * sys.float_info.epsilon) - scale = 2.0 ** 537 - return _sqrtprod(scale * x, scale * y) / scale - return h - # Improve accuracy with a differential correction. - # https://www.wolframalpha.com/input/?i=Maclaurin+series+sqrt%28h**2+%2B+x%29+at+x%3D0 - d = sumprod((x, h), (y, -h)) - return h + d / (2.0 * h) - - -# === Statistics for relations between two inputs === - -# See https://en.wikipedia.org/wiki/Covariance -# https://en.wikipedia.org/wiki/Pearson_correlation_coefficient -# https://en.wikipedia.org/wiki/Simple_linear_regression - +## Statistics for relations between two inputs ############################# def covariance(x, y, /): """Covariance @@ -1287,6 +671,7 @@ def covariance(x, y, /): -7.5 """ + # https://en.wikipedia.org/wiki/Covariance n = len(x) if len(y) != n: raise StatisticsError('covariance requires that both inputs have same number of data points') @@ -1320,7 +705,10 @@ def correlation(x, y, /, *, method='linear'): Spearman's rank correlation coefficient is appropriate for ordinal data or for continuous data that doesn't meet the linear proportion requirement for Pearson's correlation coefficient. + """ + # https://en.wikipedia.org/wiki/Pearson_correlation_coefficient + # https://en.wikipedia.org/wiki/Spearman%27s_rank_correlation_coefficient n = len(x) if len(y) != n: raise StatisticsError('correlation requires that both inputs have same number of data points') @@ -1328,18 +716,22 @@ def correlation(x, y, /, *, method='linear'): raise StatisticsError('correlation requires at least two data points') if method not in {'linear', 'ranked'}: raise ValueError(f'Unknown method: {method!r}') + if method == 'ranked': start = (n - 1) / -2 # Center rankings around zero x = _rank(x, start=start) y = _rank(y, start=start) + else: xbar = fsum(x) / n ybar = fsum(y) / n x = [xi - xbar for xi in x] y = [yi - ybar for yi in y] + sxy = sumprod(x, y) sxx = sumprod(x, x) syy = sumprod(y, y) + try: return sxy / _sqrtprod(sxx, syy) except ZeroDivisionError: @@ -1387,381 +779,317 @@ def linear_regression(x, y, /, *, proportional=False): LinearRegression(slope=2.90475..., intercept=0.0) """ + # https://en.wikipedia.org/wiki/Simple_linear_regression n = len(x) if len(y) != n: raise StatisticsError('linear regression requires that both inputs have same number of data points') if n < 2: raise StatisticsError('linear regression requires at least two data points') + if not proportional: xbar = fsum(x) / n ybar = fsum(y) / n x = [xi - xbar for xi in x] # List because used three times below y = (yi - ybar for yi in y) # Generator because only used once below + sxy = sumprod(x, y) + 0.0 # Add zero to coerce result to a float sxx = sumprod(x, x) + try: slope = sxy / sxx # equivalent to: covariance(x, y) / variance(x) except ZeroDivisionError: raise StatisticsError('x is constant') + intercept = 0.0 if proportional else ybar - slope * xbar return LinearRegression(slope=slope, intercept=intercept) -## Normal Distribution ##################################################### - +## Kernel Density Estimation ############################################### + +_kernel_specs = {} + +def register(*kernels): + "Load the kernel's pdf, cdf, invcdf, and support into _kernel_specs." + def deco(builder): + spec = dict(zip(('pdf', 'cdf', 'invcdf', 'support'), builder())) + for kernel in kernels: + _kernel_specs[kernel] = spec + return builder + return deco + +@register('normal', 'gauss') +def normal_kernel(): + sqrt2pi = sqrt(2 * pi) + neg_sqrt2 = -sqrt(2) + pdf = lambda t: exp(-1/2 * t * t) / sqrt2pi + cdf = lambda t: 1/2 * erfc(t / neg_sqrt2) + invcdf = lambda t: _normal_dist_inv_cdf(t, 0.0, 1.0) + support = None + return pdf, cdf, invcdf, support + +@register('logistic') +def logistic_kernel(): + # 1.0 / (exp(t) + 2.0 + exp(-t)) + pdf = lambda t: 1/2 / (1.0 + cosh(t)) + cdf = lambda t: 1.0 - 1.0 / (exp(t) + 1.0) + invcdf = lambda p: log(p / (1.0 - p)) + support = None + return pdf, cdf, invcdf, support + +@register('sigmoid') +def sigmoid_kernel(): + # (2/pi) / (exp(t) + exp(-t)) + c1 = 1 / pi + c2 = 2 / pi + c3 = pi / 2 + pdf = lambda t: c1 / cosh(t) + cdf = lambda t: c2 * atan(exp(t)) + invcdf = lambda p: log(tan(p * c3)) + support = None + return pdf, cdf, invcdf, support + +@register('rectangular', 'uniform') +def rectangular_kernel(): + pdf = lambda t: 1/2 + cdf = lambda t: 1/2 * t + 1/2 + invcdf = lambda p: 2.0 * p - 1.0 + support = 1.0 + return pdf, cdf, invcdf, support + +@register('triangular') +def triangular_kernel(): + pdf = lambda t: 1.0 - abs(t) + cdf = lambda t: t*t * (1/2 if t < 0.0 else -1/2) + t + 1/2 + invcdf = lambda p: sqrt(2.0*p) - 1.0 if p < 1/2 else 1.0 - sqrt(2.0 - 2.0*p) + support = 1.0 + return pdf, cdf, invcdf, support + +@register('parabolic', 'epanechnikov') +def parabolic_kernel(): + pdf = lambda t: 3/4 * (1.0 - t * t) + cdf = lambda t: sumprod((-1/4, 3/4, 1/2), (t**3, t, 1.0)) + invcdf = lambda p: 2.0 * cos((acos(2.0*p - 1.0) + pi) / 3.0) + support = 1.0 + return pdf, cdf, invcdf, support -def _normal_dist_inv_cdf(p, mu, sigma): - # There is no closed-form solution to the inverse CDF for the normal - # distribution, so we use a rational approximation instead: - # Wichura, M.J. (1988). "Algorithm AS241: The Percentage Points of the - # Normal Distribution". Applied Statistics. Blackwell Publishing. 37 - # (3): 477–484. doi:10.2307/2347330. JSTOR 2347330. - q = p - 0.5 - if fabs(q) <= 0.425: - r = 0.180625 - q * q - # Hash sum: 55.88319_28806_14901_4439 - num = (((((((2.50908_09287_30122_6727e+3 * r + - 3.34305_75583_58812_8105e+4) * r + - 6.72657_70927_00870_0853e+4) * r + - 4.59219_53931_54987_1457e+4) * r + - 1.37316_93765_50946_1125e+4) * r + - 1.97159_09503_06551_4427e+3) * r + - 1.33141_66789_17843_7745e+2) * r + - 3.38713_28727_96366_6080e+0) * q - den = (((((((5.22649_52788_52854_5610e+3 * r + - 2.87290_85735_72194_2674e+4) * r + - 3.93078_95800_09271_0610e+4) * r + - 2.12137_94301_58659_5867e+4) * r + - 5.39419_60214_24751_1077e+3) * r + - 6.87187_00749_20579_0830e+2) * r + - 4.23133_30701_60091_1252e+1) * r + - 1.0) - x = num / den - return mu + (x * sigma) - r = p if q <= 0.0 else 1.0 - p - r = sqrt(-log(r)) - if r <= 5.0: - r = r - 1.6 - # Hash sum: 49.33206_50330_16102_89036 - num = (((((((7.74545_01427_83414_07640e-4 * r + - 2.27238_44989_26918_45833e-2) * r + - 2.41780_72517_74506_11770e-1) * r + - 1.27045_82524_52368_38258e+0) * r + - 3.64784_83247_63204_60504e+0) * r + - 5.76949_72214_60691_40550e+0) * r + - 4.63033_78461_56545_29590e+0) * r + - 1.42343_71107_49683_57734e+0) - den = (((((((1.05075_00716_44416_84324e-9 * r + - 5.47593_80849_95344_94600e-4) * r + - 1.51986_66563_61645_71966e-2) * r + - 1.48103_97642_74800_74590e-1) * r + - 6.89767_33498_51000_04550e-1) * r + - 1.67638_48301_83803_84940e+0) * r + - 2.05319_16266_37758_82187e+0) * r + - 1.0) - else: - r = r - 5.0 - # Hash sum: 47.52583_31754_92896_71629 - num = (((((((2.01033_43992_92288_13265e-7 * r + - 2.71155_55687_43487_57815e-5) * r + - 1.24266_09473_88078_43860e-3) * r + - 2.65321_89526_57612_30930e-2) * r + - 2.96560_57182_85048_91230e-1) * r + - 1.78482_65399_17291_33580e+0) * r + - 5.46378_49111_64114_36990e+0) * r + - 6.65790_46435_01103_77720e+0) - den = (((((((2.04426_31033_89939_78564e-15 * r + - 1.42151_17583_16445_88870e-7) * r + - 1.84631_83175_10054_68180e-5) * r + - 7.86869_13114_56132_59100e-4) * r + - 1.48753_61290_85061_48525e-2) * r + - 1.36929_88092_27358_05310e-1) * r + - 5.99832_20655_58879_37690e-1) * r + - 1.0) - x = num / den - if q < 0.0: - x = -x - return mu + (x * sigma) +def _newton_raphson(f_inv_estimate, f, f_prime, tolerance=1e-12): + def f_inv(y): + "Return x such that f(x) ≈ y within the specified tolerance." + x = f_inv_estimate(y) + while abs(diff := f(x) - y) > tolerance: + x -= diff / f_prime(x) + return x + return f_inv +def _quartic_invcdf_estimate(p): + # A handrolled piecewise approximation. There is no magic here. + sign, p = (1.0, p) if p <= 1/2 else (-1.0, 1.0 - p) + if p < 0.0106: + return ((2.0 * p) ** 0.3838 - 1.0) * sign + x = (2.0 * p) ** 0.4258865685331 - 1.0 + if p < 0.499: + x += 0.026818732 * sin(7.101753784 * p + 2.73230839482953) + return x * sign -# If available, use C implementation -try: - from _statistics import _normal_dist_inv_cdf -except ImportError: - pass +@register('quartic', 'biweight') +def quartic_kernel(): + pdf = lambda t: 15/16 * (1.0 - t * t) ** 2 + cdf = lambda t: sumprod((3/16, -5/8, 15/16, 1/2), + (t**5, t**3, t, 1.0)) + invcdf = _newton_raphson(_quartic_invcdf_estimate, f=cdf, f_prime=pdf) + support = 1.0 + return pdf, cdf, invcdf, support +def _triweight_invcdf_estimate(p): + # A handrolled piecewise approximation. There is no magic here. + sign, p = (1.0, p) if p <= 1/2 else (-1.0, 1.0 - p) + x = (2.0 * p) ** 0.3400218741872791 - 1.0 + if 0.00001 < p < 0.499: + x -= 0.033 * sin(1.07 * tau * (p - 0.035)) + return x * sign -class NormalDist: - "Normal distribution of a random variable" - # https://en.wikipedia.org/wiki/Normal_distribution - # https://en.wikipedia.org/wiki/Variance#Properties +@register('triweight') +def triweight_kernel(): + pdf = lambda t: 35/32 * (1.0 - t * t) ** 3 + cdf = lambda t: sumprod((-5/32, 21/32, -35/32, 35/32, 1/2), + (t**7, t**5, t**3, t, 1.0)) + invcdf = _newton_raphson(_triweight_invcdf_estimate, f=cdf, f_prime=pdf) + support = 1.0 + return pdf, cdf, invcdf, support + +@register('cosine') +def cosine_kernel(): + c1 = pi / 4 + c2 = pi / 2 + pdf = lambda t: c1 * cos(c2 * t) + cdf = lambda t: 1/2 * sin(c2 * t) + 1/2 + invcdf = lambda p: 2.0 * asin(2.0 * p - 1.0) / pi + support = 1.0 + return pdf, cdf, invcdf, support + +del register, normal_kernel, logistic_kernel, sigmoid_kernel +del rectangular_kernel, triangular_kernel, parabolic_kernel +del quartic_kernel, triweight_kernel, cosine_kernel - __slots__ = { - '_mu': 'Arithmetic mean of a normal distribution', - '_sigma': 'Standard deviation of a normal distribution', - } - def __init__(self, mu=0.0, sigma=1.0): - "NormalDist where mu is the mean and sigma is the standard deviation." - if sigma < 0.0: - raise StatisticsError('sigma must be non-negative') - self._mu = float(mu) - self._sigma = float(sigma) +def kde(data, h, kernel='normal', *, cumulative=False): + """Kernel Density Estimation: Create a continuous probability density + function or cumulative distribution function from discrete samples. - @classmethod - def from_samples(cls, data): - "Make a normal distribution instance from sample data." - return cls(*_mean_stdev(data)) + The basic idea is to smooth the data using a kernel function + to help draw inferences about a population from a sample. - def samples(self, n, *, seed=None): - "Generate *n* samples for a given mean and standard deviation." - rnd = random.random if seed is None else random.Random(seed).random - inv_cdf = _normal_dist_inv_cdf - mu = self._mu - sigma = self._sigma - return [inv_cdf(rnd(), mu, sigma) for _ in repeat(None, n)] + The degree of smoothing is controlled by the scaling parameter h + which is called the bandwidth. Smaller values emphasize local + features while larger values give smoother results. - def pdf(self, x): - "Probability density function. P(x <= X < x+dx) / dx" - variance = self._sigma * self._sigma - if not variance: - raise StatisticsError('pdf() not defined when sigma is zero') - diff = x - self._mu - return exp(diff * diff / (-2.0 * variance)) / sqrt(tau * variance) + The kernel determines the relative weights of the sample data + points. Generally, the choice of kernel shape does not matter + as much as the more influential bandwidth smoothing parameter. - def cdf(self, x): - "Cumulative distribution function. P(X <= x)" - if not self._sigma: - raise StatisticsError('cdf() not defined when sigma is zero') - return 0.5 * (1.0 + erf((x - self._mu) / (self._sigma * _SQRT2))) + Kernels that give some weight to every sample point: - def inv_cdf(self, p): - """Inverse cumulative distribution function. x : P(X <= x) = p + normal (gauss) + logistic + sigmoid - Finds the value of the random variable such that the probability of - the variable being less than or equal to that value equals the given - probability. + Kernels that only give weight to sample points within + the bandwidth: - This function is also called the percent point function or quantile - function. - """ - if p <= 0.0 or p >= 1.0: - raise StatisticsError('p must be in the range 0.0 < p < 1.0') - return _normal_dist_inv_cdf(p, self._mu, self._sigma) + rectangular (uniform) + triangular + parabolic (epanechnikov) + quartic (biweight) + triweight + cosine - def quantiles(self, n=4): - """Divide into *n* continuous intervals with equal probability. + If *cumulative* is true, will return a cumulative distribution function. - Returns a list of (n - 1) cut points separating the intervals. + A StatisticsError will be raised if the data sequence is empty. - Set *n* to 4 for quartiles (the default). Set *n* to 10 for deciles. - Set *n* to 100 for percentiles which gives the 99 cuts points that - separate the normal distribution in to 100 equal sized groups. - """ - return [self.inv_cdf(i / n) for i in range(1, n)] + Example + ------- - def overlap(self, other): - """Compute the overlapping coefficient (OVL) between two normal distributions. + Given a sample of six data points, construct a continuous + function that estimates the underlying probability density: - Measures the agreement between two normal probability distributions. - Returns a value between 0.0 and 1.0 giving the overlapping area in - the two underlying probability density functions. + >>> sample = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] + >>> f_hat = kde(sample, h=1.5) - >>> N1 = NormalDist(2.4, 1.6) - >>> N2 = NormalDist(3.2, 2.0) - >>> N1.overlap(N2) - 0.8035050657330205 - """ - # See: "The overlapping coefficient as a measure of agreement between - # probability distributions and point estimation of the overlap of two - # normal densities" -- Henry F. Inman and Edwin L. Bradley Jr - # http://dx.doi.org/10.1080/03610928908830127 - if not isinstance(other, NormalDist): - raise TypeError('Expected another NormalDist instance') - X, Y = self, other - if (Y._sigma, Y._mu) < (X._sigma, X._mu): # sort to assure commutativity - X, Y = Y, X - X_var, Y_var = X.variance, Y.variance - if not X_var or not Y_var: - raise StatisticsError('overlap() not defined when sigma is zero') - dv = Y_var - X_var - dm = fabs(Y._mu - X._mu) - if not dv: - return 1.0 - erf(dm / (2.0 * X._sigma * _SQRT2)) - a = X._mu * Y_var - Y._mu * X_var - b = X._sigma * Y._sigma * sqrt(dm * dm + dv * log(Y_var / X_var)) - x1 = (a + b) / dv - x2 = (a - b) / dv - return 1.0 - (fabs(Y.cdf(x1) - X.cdf(x1)) + fabs(Y.cdf(x2) - X.cdf(x2))) + Compute the area under the curve: - def zscore(self, x): - """Compute the Standard Score. (x - mean) / stdev + >>> area = sum(f_hat(x) for x in range(-20, 20)) + >>> round(area, 4) + 1.0 - Describes *x* in terms of the number of standard deviations - above or below the mean of the normal distribution. - """ - # https://www.statisticshowto.com/probability-and-statistics/z-score/ - if not self._sigma: - raise StatisticsError('zscore() not defined when sigma is zero') - return (x - self._mu) / self._sigma + Plot the estimated probability density function at + evenly spaced points from -6 to 10: - @property - def mean(self): - "Arithmetic mean of the normal distribution." - return self._mu - - @property - def median(self): - "Return the median of the normal distribution" - return self._mu - - @property - def mode(self): - """Return the mode of the normal distribution - - The mode is the value x where which the probability density - function (pdf) takes its maximum value. - """ - return self._mu - - @property - def stdev(self): - "Standard deviation of the normal distribution." - return self._sigma - - @property - def variance(self): - "Square of the standard deviation." - return self._sigma * self._sigma - - def __add__(x1, x2): - """Add a constant or another NormalDist instance. - - If *other* is a constant, translate mu by the constant, - leaving sigma unchanged. - - If *other* is a NormalDist, add both the means and the variances. - Mathematically, this works only if the two distributions are - independent or if they are jointly normally distributed. - """ - if isinstance(x2, NormalDist): - return NormalDist(x1._mu + x2._mu, hypot(x1._sigma, x2._sigma)) - return NormalDist(x1._mu + x2, x1._sigma) - - def __sub__(x1, x2): - """Subtract a constant or another NormalDist instance. - - If *other* is a constant, translate by the constant mu, - leaving sigma unchanged. + >>> for x in range(-6, 11): + ... density = f_hat(x) + ... plot = ' ' * int(density * 400) + 'x' + ... print(f'{x:2}: {density:.3f} {plot}') + ... + -6: 0.002 x + -5: 0.009 x + -4: 0.031 x + -3: 0.070 x + -2: 0.111 x + -1: 0.125 x + 0: 0.110 x + 1: 0.086 x + 2: 0.068 x + 3: 0.059 x + 4: 0.066 x + 5: 0.082 x + 6: 0.082 x + 7: 0.058 x + 8: 0.028 x + 9: 0.009 x + 10: 0.002 x - If *other* is a NormalDist, subtract the means and add the variances. - Mathematically, this works only if the two distributions are - independent or if they are jointly normally distributed. - """ - if isinstance(x2, NormalDist): - return NormalDist(x1._mu - x2._mu, hypot(x1._sigma, x2._sigma)) - return NormalDist(x1._mu - x2, x1._sigma) + Estimate P(4.5 < X <= 7.5), the probability that a new sample value + will be between 4.5 and 7.5: - def __mul__(x1, x2): - """Multiply both mu and sigma by a constant. + >>> cdf = kde(sample, h=1.5, cumulative=True) + >>> round(cdf(7.5) - cdf(4.5), 2) + 0.22 - Used for rescaling, perhaps to change measurement units. - Sigma is scaled with the absolute value of the constant. - """ - return NormalDist(x1._mu * x2, x1._sigma * fabs(x2)) + References + ---------- - def __truediv__(x1, x2): - """Divide both mu and sigma by a constant. + Kernel density estimation and its application: + https://www.itm-conferences.org/articles/itmconf/pdf/2018/08/itmconf_sam2018_00037.pdf - Used for rescaling, perhaps to change measurement units. - Sigma is scaled with the absolute value of the constant. - """ - return NormalDist(x1._mu / x2, x1._sigma / fabs(x2)) + Kernel functions in common use: + https://en.wikipedia.org/wiki/Kernel_(statistics)#kernel_functions_in_common_use - def __pos__(x1): - "Return a copy of the instance." - return NormalDist(x1._mu, x1._sigma) + Interactive graphical demonstration and exploration: + https://demonstrations.wolfram.com/KernelDensityEstimation/ - def __neg__(x1): - "Negates mu while keeping sigma the same." - return NormalDist(-x1._mu, x1._sigma) + Kernel estimation of cumulative distribution function of a random variable with bounded support + https://www.econstor.eu/bitstream/10419/207829/1/10.21307_stattrans-2016-037.pdf - __radd__ = __add__ + """ - def __rsub__(x1, x2): - "Subtract a NormalDist from a constant or another NormalDist." - return -(x1 - x2) + n = len(data) + if not n: + raise StatisticsError('Empty data sequence') - __rmul__ = __mul__ + if not isinstance(data[0], (int, float)): + raise TypeError('Data sequence must contain ints or floats') - def __eq__(x1, x2): - "Two NormalDist objects are equal if their mu and sigma are both equal." - if not isinstance(x2, NormalDist): - return NotImplemented - return x1._mu == x2._mu and x1._sigma == x2._sigma + if h <= 0.0: + raise StatisticsError(f'Bandwidth h must be positive, not {h=!r}') - def __hash__(self): - "NormalDist objects hash equal if their mu and sigma are both equal." - return hash((self._mu, self._sigma)) + kernel_spec = _kernel_specs.get(kernel) + if kernel_spec is None: + raise StatisticsError(f'Unknown kernel name: {kernel!r}') + K = kernel_spec['pdf'] + W = kernel_spec['cdf'] + support = kernel_spec['support'] - def __repr__(self): - return f'{type(self).__name__}(mu={self._mu!r}, sigma={self._sigma!r})' + if support is None: - def __getstate__(self): - return self._mu, self._sigma + def pdf(x): + return sum(K((x - x_i) / h) for x_i in data) / (len(data) * h) - def __setstate__(self, state): - self._mu, self._sigma = state + def cdf(x): + return sum(W((x - x_i) / h) for x_i in data) / len(data) + else: -## kde_random() ############################################################## + sample = sorted(data) + bandwidth = h * support -def _newton_raphson(f_inv_estimate, f, f_prime, tolerance=1e-12): - def f_inv(y): - "Return x such that f(x) ≈ y within the specified tolerance." - x = f_inv_estimate(y) - while abs(diff := f(x) - y) > tolerance: - x -= diff / f_prime(x) - return x - return f_inv + def pdf(x): + nonlocal n, sample + if len(data) != n: + sample = sorted(data) + n = len(data) + i = bisect_left(sample, x - bandwidth) + j = bisect_right(sample, x + bandwidth) + supported = sample[i : j] + return sum(K((x - x_i) / h) for x_i in supported) / (n * h) -def _quartic_invcdf_estimate(p): - sign, p = (1.0, p) if p <= 1/2 else (-1.0, 1.0 - p) - x = (2.0 * p) ** 0.4258865685331 - 1.0 - if p >= 0.004 < 0.499: - x += 0.026818732 * sin(7.101753784 * p + 2.73230839482953) - return x * sign + def cdf(x): + nonlocal n, sample + if len(data) != n: + sample = sorted(data) + n = len(data) + i = bisect_left(sample, x - bandwidth) + j = bisect_right(sample, x + bandwidth) + supported = sample[i : j] + return sum((W((x - x_i) / h) for x_i in supported), i) / n -_quartic_invcdf = _newton_raphson( - f_inv_estimate = _quartic_invcdf_estimate, - f = lambda t: 3/16 * t**5 - 5/8 * t**3 + 15/16 * t + 1/2, - f_prime = lambda t: 15/16 * (1.0 - t * t) ** 2) + if cumulative: + cdf.__doc__ = f'CDF estimate with {h=!r} and {kernel=!r}' + return cdf -def _triweight_invcdf_estimate(p): - sign, p = (1.0, p) if p <= 1/2 else (-1.0, 1.0 - p) - x = (2.0 * p) ** 0.3400218741872791 - 1.0 - return x * sign + else: + pdf.__doc__ = f'PDF estimate with {h=!r} and {kernel=!r}' + return pdf -_triweight_invcdf = _newton_raphson( - f_inv_estimate = _triweight_invcdf_estimate, - f = lambda t: 35/32 * (-1/7*t**7 + 3/5*t**5 - t**3 + t) + 1/2, - f_prime = lambda t: 35/32 * (1.0 - t * t) ** 3) - -_kernel_invcdfs = { - 'normal': NormalDist().inv_cdf, - 'logistic': lambda p: log(p / (1 - p)), - 'sigmoid': lambda p: log(tan(p * pi/2)), - 'rectangular': lambda p: 2*p - 1, - 'parabolic': lambda p: 2 * cos((acos(2*p-1) + pi) / 3), - 'quartic': _quartic_invcdf, - 'triweight': _triweight_invcdf, - 'triangular': lambda p: sqrt(2*p) - 1 if p < 1/2 else 1 - sqrt(2 - 2*p), - 'cosine': lambda p: 2 * asin(2*p - 1) / pi, -} -_kernel_invcdfs['gauss'] = _kernel_invcdfs['normal'] -_kernel_invcdfs['uniform'] = _kernel_invcdfs['rectangular'] -_kernel_invcdfs['epanechnikov'] = _kernel_invcdfs['parabolic'] -_kernel_invcdfs['biweight'] = _kernel_invcdfs['quartic'] def kde_random(data, h, kernel='normal', *, seed=None): """Return a function that makes a random selection from the estimated @@ -1791,17 +1119,761 @@ def kde_random(data, h, kernel='normal', *, seed=None): if h <= 0.0: raise StatisticsError(f'Bandwidth h must be positive, not {h=!r}') - kernel_invcdf = _kernel_invcdfs.get(kernel) - if kernel_invcdf is None: + kernel_spec = _kernel_specs.get(kernel) + if kernel_spec is None: raise StatisticsError(f'Unknown kernel name: {kernel!r}') + invcdf = kernel_spec['invcdf'] prng = _random.Random(seed) random = prng.random choice = prng.choice def rand(): - return choice(data) + h * kernel_invcdf(random()) + return choice(data) + h * invcdf(random()) rand.__doc__ = f'Random KDE selection with {h=!r} and {kernel=!r}' return rand + + +## Quantiles ############################################################### + +# There is no one perfect way to compute quantiles. Here we offer +# two methods that serve common needs. Most other packages +# surveyed offered at least one or both of these two, making them +# "standard" in the sense of "widely-adopted and reproducible". +# They are also easy to explain, easy to compute manually, and have +# straight-forward interpretations that aren't surprising. + +# The default method is known as "R6", "PERCENTILE.EXC", or "expected +# value of rank order statistics". The alternative method is known as +# "R7", "PERCENTILE.INC", or "mode of rank order statistics". + +# For sample data where there is a positive probability for values +# beyond the range of the data, the R6 exclusive method is a +# reasonable choice. Consider a random sample of nine values from a +# population with a uniform distribution from 0.0 to 1.0. The +# distribution of the third ranked sample point is described by +# betavariate(alpha=3, beta=7) which has mode=0.250, median=0.286, and +# mean=0.300. Only the latter (which corresponds with R6) gives the +# desired cut point with 30% of the population falling below that +# value, making it comparable to a result from an inv_cdf() function. +# The R6 exclusive method is also idempotent. + +# For describing population data where the end points are known to +# be included in the data, the R7 inclusive method is a reasonable +# choice. Instead of the mean, it uses the mode of the beta +# distribution for the interior points. Per Hyndman & Fan, "One nice +# property is that the vertices of Q7(p) divide the range into n - 1 +# intervals, and exactly 100p% of the intervals lie to the left of +# Q7(p) and 100(1 - p)% of the intervals lie to the right of Q7(p)." + +# If needed, other methods could be added. However, for now, the +# position is that fewer options make for easier choices and that +# external packages can be used for anything more advanced. + +def quantiles(data, *, n=4, method='exclusive'): + """Divide *data* into *n* continuous intervals with equal probability. + + Returns a list of (n - 1) cut points separating the intervals. + + Set *n* to 4 for quartiles (the default). Set *n* to 10 for deciles. + Set *n* to 100 for percentiles which gives the 99 cuts points that + separate *data* in to 100 equal sized groups. + + The *data* can be any iterable containing sample. + The cut points are linearly interpolated between data points. + + If *method* is set to *inclusive*, *data* is treated as population + data. The minimum value is treated as the 0th percentile and the + maximum value is treated as the 100th percentile. + + """ + if n < 1: + raise StatisticsError('n must be at least 1') + + data = sorted(data) + + ld = len(data) + if ld < 2: + if ld == 1: + return data * (n - 1) + raise StatisticsError('must have at least one data point') + + if method == 'inclusive': + m = ld - 1 + result = [] + for i in range(1, n): + j, delta = divmod(i * m, n) + interpolated = (data[j] * (n - delta) + data[j + 1] * delta) / n + result.append(interpolated) + return result + + if method == 'exclusive': + m = ld + 1 + result = [] + for i in range(1, n): + j = i * m // n # rescale i to m/n + j = 1 if j < 1 else ld-1 if j > ld-1 else j # clamp to 1 .. ld-1 + delta = i*m - j*n # exact integer math + interpolated = (data[j - 1] * (n - delta) + data[j] * delta) / n + result.append(interpolated) + return result + + raise ValueError(f'Unknown method: {method!r}') + + +## Normal Distribution ##################################################### + +class NormalDist: + "Normal distribution of a random variable" + # https://en.wikipedia.org/wiki/Normal_distribution + # https://en.wikipedia.org/wiki/Variance#Properties + + __slots__ = { + '_mu': 'Arithmetic mean of a normal distribution', + '_sigma': 'Standard deviation of a normal distribution', + } + + def __init__(self, mu=0.0, sigma=1.0): + "NormalDist where mu is the mean and sigma is the standard deviation." + if sigma < 0.0: + raise StatisticsError('sigma must be non-negative') + self._mu = float(mu) + self._sigma = float(sigma) + + @classmethod + def from_samples(cls, data): + "Make a normal distribution instance from sample data." + return cls(*_mean_stdev(data)) + + def samples(self, n, *, seed=None): + "Generate *n* samples for a given mean and standard deviation." + rnd = random.random if seed is None else random.Random(seed).random + inv_cdf = _normal_dist_inv_cdf + mu = self._mu + sigma = self._sigma + return [inv_cdf(rnd(), mu, sigma) for _ in repeat(None, n)] + + def pdf(self, x): + "Probability density function. P(x <= X < x+dx) / dx" + variance = self._sigma * self._sigma + if not variance: + raise StatisticsError('pdf() not defined when sigma is zero') + diff = x - self._mu + return exp(diff * diff / (-2.0 * variance)) / sqrt(tau * variance) + + def cdf(self, x): + "Cumulative distribution function. P(X <= x)" + if not self._sigma: + raise StatisticsError('cdf() not defined when sigma is zero') + return 0.5 * erfc((self._mu - x) / (self._sigma * _SQRT2)) + + def inv_cdf(self, p): + """Inverse cumulative distribution function. x : P(X <= x) = p + + Finds the value of the random variable such that the probability of + the variable being less than or equal to that value equals the given + probability. + + This function is also called the percent point function or quantile + function. + """ + if p <= 0.0 or p >= 1.0: + raise StatisticsError('p must be in the range 0.0 < p < 1.0') + return _normal_dist_inv_cdf(p, self._mu, self._sigma) + + def quantiles(self, n=4): + """Divide into *n* continuous intervals with equal probability. + + Returns a list of (n - 1) cut points separating the intervals. + + Set *n* to 4 for quartiles (the default). Set *n* to 10 for deciles. + Set *n* to 100 for percentiles which gives the 99 cuts points that + separate the normal distribution in to 100 equal sized groups. + """ + return [self.inv_cdf(i / n) for i in range(1, n)] + + def overlap(self, other): + """Compute the overlapping coefficient (OVL) between two normal distributions. + + Measures the agreement between two normal probability distributions. + Returns a value between 0.0 and 1.0 giving the overlapping area in + the two underlying probability density functions. + + >>> N1 = NormalDist(2.4, 1.6) + >>> N2 = NormalDist(3.2, 2.0) + >>> N1.overlap(N2) + 0.8035050657330205 + """ + # See: "The overlapping coefficient as a measure of agreement between + # probability distributions and point estimation of the overlap of two + # normal densities" -- Henry F. Inman and Edwin L. Bradley Jr + # http://dx.doi.org/10.1080/03610928908830127 + if not isinstance(other, NormalDist): + raise TypeError('Expected another NormalDist instance') + X, Y = self, other + if (Y._sigma, Y._mu) < (X._sigma, X._mu): # sort to assure commutativity + X, Y = Y, X + X_var, Y_var = X.variance, Y.variance + if not X_var or not Y_var: + raise StatisticsError('overlap() not defined when sigma is zero') + dv = Y_var - X_var + dm = fabs(Y._mu - X._mu) + if not dv: + return erfc(dm / (2.0 * X._sigma * _SQRT2)) + a = X._mu * Y_var - Y._mu * X_var + b = X._sigma * Y._sigma * sqrt(dm * dm + dv * log(Y_var / X_var)) + x1 = (a + b) / dv + x2 = (a - b) / dv + return 1.0 - (fabs(Y.cdf(x1) - X.cdf(x1)) + fabs(Y.cdf(x2) - X.cdf(x2))) + + def zscore(self, x): + """Compute the Standard Score. (x - mean) / stdev + + Describes *x* in terms of the number of standard deviations + above or below the mean of the normal distribution. + """ + # https://www.statisticshowto.com/probability-and-statistics/z-score/ + if not self._sigma: + raise StatisticsError('zscore() not defined when sigma is zero') + return (x - self._mu) / self._sigma + + @property + def mean(self): + "Arithmetic mean of the normal distribution." + return self._mu + + @property + def median(self): + "Return the median of the normal distribution" + return self._mu + + @property + def mode(self): + """Return the mode of the normal distribution + + The mode is the value x where which the probability density + function (pdf) takes its maximum value. + """ + return self._mu + + @property + def stdev(self): + "Standard deviation of the normal distribution." + return self._sigma + + @property + def variance(self): + "Square of the standard deviation." + return self._sigma * self._sigma + + def __add__(x1, x2): + """Add a constant or another NormalDist instance. + + If *other* is a constant, translate mu by the constant, + leaving sigma unchanged. + + If *other* is a NormalDist, add both the means and the variances. + Mathematically, this works only if the two distributions are + independent or if they are jointly normally distributed. + """ + if isinstance(x2, NormalDist): + return NormalDist(x1._mu + x2._mu, hypot(x1._sigma, x2._sigma)) + return NormalDist(x1._mu + x2, x1._sigma) + + def __sub__(x1, x2): + """Subtract a constant or another NormalDist instance. + + If *other* is a constant, translate by the constant mu, + leaving sigma unchanged. + + If *other* is a NormalDist, subtract the means and add the variances. + Mathematically, this works only if the two distributions are + independent or if they are jointly normally distributed. + """ + if isinstance(x2, NormalDist): + return NormalDist(x1._mu - x2._mu, hypot(x1._sigma, x2._sigma)) + return NormalDist(x1._mu - x2, x1._sigma) + + def __mul__(x1, x2): + """Multiply both mu and sigma by a constant. + + Used for rescaling, perhaps to change measurement units. + Sigma is scaled with the absolute value of the constant. + """ + return NormalDist(x1._mu * x2, x1._sigma * fabs(x2)) + + def __truediv__(x1, x2): + """Divide both mu and sigma by a constant. + + Used for rescaling, perhaps to change measurement units. + Sigma is scaled with the absolute value of the constant. + """ + return NormalDist(x1._mu / x2, x1._sigma / fabs(x2)) + + def __pos__(x1): + "Return a copy of the instance." + return NormalDist(x1._mu, x1._sigma) + + def __neg__(x1): + "Negates mu while keeping sigma the same." + return NormalDist(-x1._mu, x1._sigma) + + __radd__ = __add__ + + def __rsub__(x1, x2): + "Subtract a NormalDist from a constant or another NormalDist." + return -(x1 - x2) + + __rmul__ = __mul__ + + def __eq__(x1, x2): + "Two NormalDist objects are equal if their mu and sigma are both equal." + if not isinstance(x2, NormalDist): + return NotImplemented + return x1._mu == x2._mu and x1._sigma == x2._sigma + + def __hash__(self): + "NormalDist objects hash equal if their mu and sigma are both equal." + return hash((self._mu, self._sigma)) + + def __repr__(self): + return f'{type(self).__name__}(mu={self._mu!r}, sigma={self._sigma!r})' + + def __getstate__(self): + return self._mu, self._sigma + + def __setstate__(self, state): + self._mu, self._sigma = state + + +## Private utilities ####################################################### + +def _sum(data): + """_sum(data) -> (type, sum, count) + + Return a high-precision sum of the given numeric data as a fraction, + together with the type to be converted to and the count of items. + + Examples + -------- + + >>> _sum([3, 2.25, 4.5, -0.5, 0.25]) + (, Fraction(19, 2), 5) + + Some sources of round-off error will be avoided: + + # Built-in sum returns zero. + >>> _sum([1e50, 1, -1e50] * 1000) + (, Fraction(1000, 1), 3000) + + Fractions and Decimals are also supported: + + >>> from fractions import Fraction as F + >>> _sum([F(2, 3), F(7, 5), F(1, 4), F(5, 6)]) + (, Fraction(63, 20), 4) + + >>> from decimal import Decimal as D + >>> data = [D("0.1375"), D("0.2108"), D("0.3061"), D("0.0419")] + >>> _sum(data) + (, Fraction(6963, 10000), 4) + + Mixed types are currently treated as an error, except that int is + allowed. + + """ + count = 0 + types = set() + types_add = types.add + partials = {} + partials_get = partials.get + + for typ, values in groupby(data, type): + types_add(typ) + for n, d in map(_exact_ratio, values): + count += 1 + partials[d] = partials_get(d, 0) + n + + if None in partials: + # The sum will be a NAN or INF. We can ignore all the finite + # partials, and just look at this special one. + total = partials[None] + assert not _isfinite(total) + else: + # Sum all the partial sums using builtin sum. + total = sum(Fraction(n, d) for d, n in partials.items()) + + T = reduce(_coerce, types, int) # or raise TypeError + return (T, total, count) + + +def _ss(data, c=None): + """Return the exact mean and sum of square deviations of sequence data. + + Calculations are done in a single pass, allowing the input to be an iterator. + + If given *c* is used the mean; otherwise, it is calculated from the data. + Use the *c* argument with care, as it can lead to garbage results. + + """ + if c is not None: + T, ssd, count = _sum((d := x - c) * d for x in data) + return (T, ssd, c, count) + + count = 0 + types = set() + types_add = types.add + sx_partials = defaultdict(int) + sxx_partials = defaultdict(int) + + for typ, values in groupby(data, type): + types_add(typ) + for n, d in map(_exact_ratio, values): + count += 1 + sx_partials[d] += n + sxx_partials[d] += n * n + + if not count: + ssd = c = Fraction(0) + + elif None in sx_partials: + # The sum will be a NAN or INF. We can ignore all the finite + # partials, and just look at this special one. + ssd = c = sx_partials[None] + assert not _isfinite(ssd) + + else: + sx = sum(Fraction(n, d) for d, n in sx_partials.items()) + sxx = sum(Fraction(n, d*d) for d, n in sxx_partials.items()) + # This formula has poor numeric properties for floats, + # but with fractions it is exact. + ssd = (count * sxx - sx * sx) / count + c = sx / count + + T = reduce(_coerce, types, int) # or raise TypeError + return (T, ssd, c, count) + + +def _isfinite(x): + try: + return x.is_finite() # Likely a Decimal. + except AttributeError: + return math.isfinite(x) # Coerces to float first. + + +def _coerce(T, S): + """Coerce types T and S to a common type, or raise TypeError. + + Coercion rules are currently an implementation detail. See the CoerceTest + test class in test_statistics for details. + + """ + # See http://bugs.python.org/issue24068. + assert T is not bool, "initial type T is bool" + # If the types are the same, no need to coerce anything. Put this + # first, so that the usual case (no coercion needed) happens as soon + # as possible. + if T is S: return T + # Mixed int & other coerce to the other type. + if S is int or S is bool: return T + if T is int: return S + # If one is a (strict) subclass of the other, coerce to the subclass. + if issubclass(S, T): return S + if issubclass(T, S): return T + # Ints coerce to the other type. + if issubclass(T, int): return S + if issubclass(S, int): return T + # Mixed fraction & float coerces to float (or float subclass). + if issubclass(T, Fraction) and issubclass(S, float): + return S + if issubclass(T, float) and issubclass(S, Fraction): + return T + # Any other combination is disallowed. + msg = "don't know how to coerce %s and %s" + raise TypeError(msg % (T.__name__, S.__name__)) + + +def _exact_ratio(x): + """Return Real number x to exact (numerator, denominator) pair. + + >>> _exact_ratio(0.25) + (1, 4) + + x is expected to be an int, Fraction, Decimal or float. + + """ + try: + return x.as_integer_ratio() + except AttributeError: + pass + except (OverflowError, ValueError): + # float NAN or INF. + assert not _isfinite(x) + return (x, None) + + try: + # x may be an Integral ABC. + return (x.numerator, x.denominator) + except AttributeError: + msg = f"can't convert type '{type(x).__name__}' to numerator/denominator" + raise TypeError(msg) + + +def _convert(value, T): + """Convert value to given numeric type T.""" + if type(value) is T: + # This covers the cases where T is Fraction, or where value is + # a NAN or INF (Decimal or float). + return value + + if issubclass(T, int) and value.denominator != 1: + T = float + + try: + # FIXME: what do we do if this overflows? + return T(value) + except TypeError: + if issubclass(T, Decimal): + return T(value.numerator) / T(value.denominator) + else: + raise + + +def _fail_neg(values, errmsg='negative value'): + """Iterate over values, failing if any are less than zero.""" + for x in values: + if x < 0: + raise StatisticsError(errmsg) + yield x + + +def _rank(data, /, *, key=None, reverse=False, ties='average', start=1) -> list[float]: + """Rank order a dataset. The lowest value has rank 1. + + Ties are averaged so that equal values receive the same rank: + + >>> data = [31, 56, 31, 25, 75, 18] + >>> _rank(data) + [3.5, 5.0, 3.5, 2.0, 6.0, 1.0] + + The operation is idempotent: + + >>> _rank([3.5, 5.0, 3.5, 2.0, 6.0, 1.0]) + [3.5, 5.0, 3.5, 2.0, 6.0, 1.0] + + It is possible to rank the data in reverse order so that the + highest value has rank 1. Also, a key-function can extract + the field to be ranked: + + >>> goals = [('eagles', 45), ('bears', 48), ('lions', 44)] + >>> _rank(goals, key=itemgetter(1), reverse=True) + [2.0, 1.0, 3.0] + + Ranks are conventionally numbered starting from one; however, + setting *start* to zero allows the ranks to be used as array indices: + + >>> prize = ['Gold', 'Silver', 'Bronze', 'Certificate'] + >>> scores = [8.1, 7.3, 9.4, 8.3] + >>> [prize[int(i)] for i in _rank(scores, start=0, reverse=True)] + ['Bronze', 'Certificate', 'Gold', 'Silver'] + + """ + # If this function becomes public at some point, more thought + # needs to be given to the signature. A list of ints is + # plausible when ties is "min" or "max". When ties is "average", + # either list[float] or list[Fraction] is plausible. + + # Default handling of ties matches scipy.stats.mstats.spearmanr. + if ties != 'average': + raise ValueError(f'Unknown tie resolution method: {ties!r}') + if key is not None: + data = map(key, data) + val_pos = sorted(zip(data, count()), reverse=reverse) + i = start - 1 + result = [0] * len(val_pos) + for _, g in groupby(val_pos, key=itemgetter(0)): + group = list(g) + size = len(group) + rank = i + (size + 1) / 2 + for value, orig_pos in group: + result[orig_pos] = rank + i += size + return result + + +def _integer_sqrt_of_frac_rto(n: int, m: int) -> int: + """Square root of n/m, rounded to the nearest integer using round-to-odd.""" + # Reference: https://www.lri.fr/~melquion/doc/05-imacs17_1-expose.pdf + a = math.isqrt(n // m) + return a | (a*a*m != n) + + +# For 53 bit precision floats, the bit width used in +# _float_sqrt_of_frac() is 109. +_sqrt_bit_width: int = 2 * sys.float_info.mant_dig + 3 + + +def _float_sqrt_of_frac(n: int, m: int) -> float: + """Square root of n/m as a float, correctly rounded.""" + # See principle and proof sketch at: https://bugs.python.org/msg407078 + q = (n.bit_length() - m.bit_length() - _sqrt_bit_width) // 2 + if q >= 0: + numerator = _integer_sqrt_of_frac_rto(n, m << 2 * q) << q + denominator = 1 + else: + numerator = _integer_sqrt_of_frac_rto(n << -2 * q, m) + denominator = 1 << -q + return numerator / denominator # Convert to float + + +def _decimal_sqrt_of_frac(n: int, m: int) -> Decimal: + """Square root of n/m as a Decimal, correctly rounded.""" + # Premise: For decimal, computing (n/m).sqrt() can be off + # by 1 ulp from the correctly rounded result. + # Method: Check the result, moving up or down a step if needed. + if n <= 0: + if not n: + return Decimal('0.0') + n, m = -n, -m + + root = (Decimal(n) / Decimal(m)).sqrt() + nr, dr = root.as_integer_ratio() + + plus = root.next_plus() + np, dp = plus.as_integer_ratio() + # test: n / m > ((root + plus) / 2) ** 2 + if 4 * n * (dr*dp)**2 > m * (dr*np + dp*nr)**2: + return plus + + minus = root.next_minus() + nm, dm = minus.as_integer_ratio() + # test: n / m < ((root + minus) / 2) ** 2 + if 4 * n * (dr*dm)**2 < m * (dr*nm + dm*nr)**2: + return minus + + return root + + +def _mean_stdev(data): + """In one pass, compute the mean and sample standard deviation as floats.""" + T, ss, xbar, n = _ss(data) + if n < 2: + raise StatisticsError('stdev requires at least two data points') + mss = ss / (n - 1) + try: + return float(xbar), _float_sqrt_of_frac(mss.numerator, mss.denominator) + except AttributeError: + # Handle Nans and Infs gracefully + return float(xbar), float(xbar) / float(ss) + + +def _sqrtprod(x: float, y: float) -> float: + "Return sqrt(x * y) computed with improved accuracy and without overflow/underflow." + + h = sqrt(x * y) + + if not isfinite(h): + if isinf(h) and not isinf(x) and not isinf(y): + # Finite inputs overflowed, so scale down, and recompute. + scale = 2.0 ** -512 # sqrt(1 / sys.float_info.max) + return _sqrtprod(scale * x, scale * y) / scale + return h + + if not h: + if x and y: + # Non-zero inputs underflowed, so scale up, and recompute. + # Scale: 1 / sqrt(sys.float_info.min * sys.float_info.epsilon) + scale = 2.0 ** 537 + return _sqrtprod(scale * x, scale * y) / scale + return h + + # Improve accuracy with a differential correction. + # https://www.wolframalpha.com/input/?i=Maclaurin+series+sqrt%28h**2+%2B+x%29+at+x%3D0 + d = sumprod((x, h), (y, -h)) + return h + d / (2.0 * h) + + +def _normal_dist_inv_cdf(p, mu, sigma): + # There is no closed-form solution to the inverse CDF for the normal + # distribution, so we use a rational approximation instead: + # Wichura, M.J. (1988). "Algorithm AS241: The Percentage Points of the + # Normal Distribution". Applied Statistics. Blackwell Publishing. 37 + # (3): 477–484. doi:10.2307/2347330. JSTOR 2347330. + q = p - 0.5 + + if fabs(q) <= 0.425: + r = 0.180625 - q * q + # Hash sum: 55.88319_28806_14901_4439 + num = (((((((2.50908_09287_30122_6727e+3 * r + + 3.34305_75583_58812_8105e+4) * r + + 6.72657_70927_00870_0853e+4) * r + + 4.59219_53931_54987_1457e+4) * r + + 1.37316_93765_50946_1125e+4) * r + + 1.97159_09503_06551_4427e+3) * r + + 1.33141_66789_17843_7745e+2) * r + + 3.38713_28727_96366_6080e+0) * q + den = (((((((5.22649_52788_52854_5610e+3 * r + + 2.87290_85735_72194_2674e+4) * r + + 3.93078_95800_09271_0610e+4) * r + + 2.12137_94301_58659_5867e+4) * r + + 5.39419_60214_24751_1077e+3) * r + + 6.87187_00749_20579_0830e+2) * r + + 4.23133_30701_60091_1252e+1) * r + + 1.0) + x = num / den + return mu + (x * sigma) + + r = p if q <= 0.0 else 1.0 - p + r = sqrt(-log(r)) + if r <= 5.0: + r = r - 1.6 + # Hash sum: 49.33206_50330_16102_89036 + num = (((((((7.74545_01427_83414_07640e-4 * r + + 2.27238_44989_26918_45833e-2) * r + + 2.41780_72517_74506_11770e-1) * r + + 1.27045_82524_52368_38258e+0) * r + + 3.64784_83247_63204_60504e+0) * r + + 5.76949_72214_60691_40550e+0) * r + + 4.63033_78461_56545_29590e+0) * r + + 1.42343_71107_49683_57734e+0) + den = (((((((1.05075_00716_44416_84324e-9 * r + + 5.47593_80849_95344_94600e-4) * r + + 1.51986_66563_61645_71966e-2) * r + + 1.48103_97642_74800_74590e-1) * r + + 6.89767_33498_51000_04550e-1) * r + + 1.67638_48301_83803_84940e+0) * r + + 2.05319_16266_37758_82187e+0) * r + + 1.0) + else: + r = r - 5.0 + # Hash sum: 47.52583_31754_92896_71629 + num = (((((((2.01033_43992_92288_13265e-7 * r + + 2.71155_55687_43487_57815e-5) * r + + 1.24266_09473_88078_43860e-3) * r + + 2.65321_89526_57612_30930e-2) * r + + 2.96560_57182_85048_91230e-1) * r + + 1.78482_65399_17291_33580e+0) * r + + 5.46378_49111_64114_36990e+0) * r + + 6.65790_46435_01103_77720e+0) + den = (((((((2.04426_31033_89939_78564e-15 * r + + 1.42151_17583_16445_88870e-7) * r + + 1.84631_83175_10054_68180e-5) * r + + 7.86869_13114_56132_59100e-4) * r + + 1.48753_61290_85061_48525e-2) * r + + 1.36929_88092_27358_05310e-1) * r + + 5.99832_20655_58879_37690e-1) * r + + 1.0) + + x = num / den + if q < 0.0: + x = -x + + return mu + (x * sigma) + + +# If available, use C implementation +try: + from _statistics import _normal_dist_inv_cdf +except ImportError: + pass diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 885f0092b53..578d7b95d05 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -79,7 +79,7 @@ if _mswindows: import _winapi - from _winapi import (CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP, + from _winapi import (CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP, # noqa: F401 STD_INPUT_HANDLE, STD_OUTPUT_HANDLE, STD_ERROR_HANDLE, SW_HIDE, STARTF_USESTDHANDLES, STARTF_USESHOWWINDOW, @@ -752,7 +752,6 @@ def _use_posix_spawn(): # These are primarily fail-safe knobs for negatives. A True value does not # guarantee the given libc/syscall API will be used. _USE_POSIX_SPAWN = _use_posix_spawn() -_USE_VFORK = True _HAVE_POSIX_SPAWN_CLOSEFROM = hasattr(os, 'POSIX_SPAWN_CLOSEFROM') @@ -1125,10 +1124,9 @@ def __exit__(self, exc_type, value, traceback): except TimeoutExpired: pass self._sigint_wait_secs = 0 # Note that this has been done. - return # resume the KeyboardInterrupt - - # Wait for the process to terminate, to avoid zombies. - self.wait() + else: + # Wait for the process to terminate, to avoid zombies. + self.wait() def __del__(self, _maxsize=sys.maxsize, _warn=warnings.warn): if not self._child_created: @@ -1927,7 +1925,7 @@ def _execute_child(self, args, executable, preexec_fn, close_fds, errpipe_read, errpipe_write, restore_signals, start_new_session, process_group, gid, gids, uid, umask, - preexec_fn, _USE_VFORK) + preexec_fn) self._child_created = True finally: # be sure the FD is closed no matter what @@ -2143,7 +2141,7 @@ def _communicate(self, input, endtime, orig_timeout): while selector.get_map(): timeout = self._remaining_time(endtime) - if timeout is not None and timeout < 0: + if timeout is not None and timeout <= 0: self._check_timeout(endtime, orig_timeout, stdout, stderr, skip_check_and_raise=True) diff --git a/Lib/symtable.py b/Lib/symtable.py index 672ec0ce1ff..7a30e1ac4ca 100644 --- a/Lib/symtable.py +++ b/Lib/symtable.py @@ -1,9 +1,16 @@ """Interface to the compiler's internal symbol tables""" import _symtable -from _symtable import (USE, DEF_GLOBAL, DEF_NONLOCAL, DEF_LOCAL, DEF_PARAM, - DEF_IMPORT, DEF_BOUND, DEF_ANNOT, SCOPE_OFF, SCOPE_MASK, FREE, - LOCAL, GLOBAL_IMPLICIT, GLOBAL_EXPLICIT, CELL) +from _symtable import ( + USE, + DEF_GLOBAL, # noqa: F401 + DEF_NONLOCAL, DEF_LOCAL, + DEF_PARAM, DEF_TYPE_PARAM, DEF_FREE_CLASS, + DEF_IMPORT, DEF_BOUND, DEF_ANNOT, + DEF_COMP_ITER, DEF_COMP_CELL, + SCOPE_OFF, SCOPE_MASK, + FREE, LOCAL, GLOBAL_IMPLICIT, GLOBAL_EXPLICIT, CELL +) import weakref from enum import StrEnum @@ -165,6 +172,10 @@ def get_children(self): for st in self._table.children] +def _get_scope(flags): # like _PyST_GetScope() + return (flags >> SCOPE_OFF) & SCOPE_MASK + + class Function(SymbolTable): # Default values for instance variables @@ -190,7 +201,7 @@ def get_locals(self): """ if self.__locals is None: locs = (LOCAL, CELL) - test = lambda x: ((x >> SCOPE_OFF) & SCOPE_MASK) in locs + test = lambda x: _get_scope(x) in locs self.__locals = self.__idents_matching(test) return self.__locals @@ -199,7 +210,7 @@ def get_globals(self): """ if self.__globals is None: glob = (GLOBAL_IMPLICIT, GLOBAL_EXPLICIT) - test = lambda x:((x >> SCOPE_OFF) & SCOPE_MASK) in glob + test = lambda x: _get_scope(x) in glob self.__globals = self.__idents_matching(test) return self.__globals @@ -214,7 +225,7 @@ def get_frees(self): """Return a tuple of free variables in the function. """ if self.__frees is None: - is_free = lambda x:((x >> SCOPE_OFF) & SCOPE_MASK) == FREE + is_free = lambda x: _get_scope(x) == FREE self.__frees = self.__idents_matching(is_free) return self.__frees @@ -226,6 +237,12 @@ class Class(SymbolTable): def get_methods(self): """Return a tuple of methods declared in the class. """ + import warnings + typename = f'{self.__class__.__module__}.{self.__class__.__name__}' + warnings.warn(f'{typename}.get_methods() is deprecated ' + f'and will be removed in Python 3.16.', + DeprecationWarning, stacklevel=2) + if self.__methods is None: d = {} @@ -268,7 +285,7 @@ class Symbol: def __init__(self, name, flags, namespaces=None, *, module_scope=False): self.__name = name self.__flags = flags - self.__scope = (flags >> SCOPE_OFF) & SCOPE_MASK # like PyST_GetScope() + self.__scope = _get_scope(flags) self.__namespaces = namespaces or () self.__module_scope = module_scope @@ -293,13 +310,18 @@ def is_referenced(self): """Return *True* if the symbol is used in its block. """ - return bool(self.__flags & _symtable.USE) + return bool(self.__flags & USE) def is_parameter(self): """Return *True* if the symbol is a parameter. """ return bool(self.__flags & DEF_PARAM) + def is_type_parameter(self): + """Return *True* if the symbol is a type parameter. + """ + return bool(self.__flags & DEF_TYPE_PARAM) + def is_global(self): """Return *True* if the symbol is global. """ @@ -332,6 +354,11 @@ def is_free(self): """ return bool(self.__scope == FREE) + def is_free_class(self): + """Return *True* if a class-scoped symbol is free from + the perspective of a method.""" + return bool(self.__flags & DEF_FREE_CLASS) + def is_imported(self): """Return *True* if the symbol is created from an import statement. @@ -342,6 +369,16 @@ def is_assigned(self): """Return *True* if a symbol is assigned to.""" return bool(self.__flags & DEF_LOCAL) + def is_comp_iter(self): + """Return *True* if the symbol is a comprehension iteration variable. + """ + return bool(self.__flags & DEF_COMP_ITER) + + def is_comp_cell(self): + """Return *True* if the symbol is a cell in an inlined comprehension. + """ + return bool(self.__flags & DEF_COMP_CELL) + def is_namespace(self): """Returns *True* if name binding introduces new namespace. diff --git a/Lib/sysconfig/__init__.py b/Lib/sysconfig/__init__.py index 8365236f61c..ae83243fa3e 100644 --- a/Lib/sysconfig/__init__.py +++ b/Lib/sysconfig/__init__.py @@ -116,8 +116,10 @@ def _getuserbase(): if env_base: return env_base - # Emscripten, iOS, tvOS, VxWorks, WASI, and watchOS have no home directories - if sys.platform in {"emscripten", "ios", "tvos", "vxworks", "wasi", "watchos"}: + # Emscripten, iOS, tvOS, VxWorks, WASI, and watchOS have no home directories. + # Use _PYTHON_HOST_PLATFORM to get the correct platform when cross-compiling. + system_name = os.environ.get('_PYTHON_HOST_PLATFORM', sys.platform).split('-')[0] + if system_name in {"emscripten", "ios", "tvos", "vxworks", "wasi", "watchos"}: return None def joinuser(*args): @@ -323,17 +325,37 @@ def get_default_scheme(): def get_makefile_filename(): """Return the path of the Makefile.""" + + # GH-127429: When cross-compiling, use the Makefile from the target, instead of the host Python. + if cross_base := os.environ.get('_PYTHON_PROJECT_BASE'): + return os.path.join(cross_base, 'Makefile') + if _PYTHON_BUILD: return os.path.join(_PROJECT_BASE, "Makefile") + if hasattr(sys, 'abiflags'): config_dir_name = f'config-{_PY_VERSION_SHORT}{sys.abiflags}' else: config_dir_name = 'config' + if hasattr(sys.implementation, '_multiarch'): config_dir_name += f'-{sys.implementation._multiarch}' + return os.path.join(get_path('stdlib'), config_dir_name, 'Makefile') +def _import_from_directory(path, name): + if name not in sys.modules: + import importlib.machinery + import importlib.util + + spec = importlib.machinery.PathFinder.find_spec(name, [path]) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + sys.modules[name] = module + return sys.modules[name] + + def _get_sysconfigdata_name(): multiarch = getattr(sys.implementation, '_multiarch', '') return os.environ.get( @@ -341,26 +363,34 @@ def _get_sysconfigdata_name(): f'_sysconfigdata_{sys.abiflags}_{sys.platform}_{multiarch}', ) + +def _get_sysconfigdata(): + import importlib + + name = _get_sysconfigdata_name() + path = os.environ.get('_PYTHON_SYSCONFIGDATA_PATH') + module = _import_from_directory(path, name) if path else importlib.import_module(name) + + return module.build_time_vars + + +def _installation_is_relocated(): + """Is the Python installation running from a different prefix than what was targetted when building?""" + if os.name != 'posix': + raise NotImplementedError('sysconfig._installation_is_relocated() is currently only supported on POSIX') + + data = _get_sysconfigdata() + return ( + data['prefix'] != getattr(sys, 'base_prefix', '') + or data['exec_prefix'] != getattr(sys, 'base_exec_prefix', '') + ) + + def _init_posix(vars): """Initialize the module as appropriate for POSIX systems.""" - # _sysconfigdata is generated at build time, see _generate_posix_vars() - name = _get_sysconfigdata_name() + # GH-126920: Make sure we don't overwrite any of the keys already set + vars.update(_get_sysconfigdata() | vars) - # For cross builds, the path to the target's sysconfigdata must be specified - # so it can be imported. It cannot be in PYTHONPATH, as foreign modules in - # sys.path can cause crashes when loaded by the host interpreter. - # Rely on truthiness as a valueless env variable is still an empty string. - # See OS X note in _generate_posix_vars re _sysconfigdata. - if (path := os.environ.get('_PYTHON_SYSCONFIGDATA_PATH')): - from importlib.machinery import FileFinder, SourceFileLoader, SOURCE_SUFFIXES - from importlib.util import module_from_spec - spec = FileFinder(path, (SourceFileLoader, SOURCE_SUFFIXES)).find_spec(name) - _temp = module_from_spec(spec) - spec.loader.exec_module(_temp) - else: - _temp = __import__(name, globals(), locals(), ['build_time_vars'], 0) - build_time_vars = _temp.build_time_vars - vars.update(build_time_vars) def _init_non_posix(vars): """Initialize the module as appropriate for NT""" @@ -371,9 +401,20 @@ def _init_non_posix(vars): vars['BINLIBDEST'] = get_path('platstdlib') vars['INCLUDEPY'] = get_path('include') - # Add EXT_SUFFIX, SOABI, and Py_GIL_DISABLED + # Add EXT_SUFFIX, SOABI, Py_DEBUG, and Py_GIL_DISABLED vars.update(_sysconfig.config_vars()) + # NOTE: ABIFLAGS is only an emulated value. It is not present during build + # on Windows. sys.abiflags is absent on Windows and vars['abiflags'] + # is already widely used to calculate paths, so it should remain an + # empty string. + vars['ABIFLAGS'] = ''.join( + ( + 't' if vars['Py_GIL_DISABLED'] else '', + '_d' if vars['Py_DEBUG'] else '', + ), + ) + vars['LIBDIR'] = _safe_realpath(os.path.join(get_config_var('installed_base'), 'libs')) if hasattr(sys, 'dllhandle'): dllhandle = _winapi.GetModuleFileName(sys.dllhandle) @@ -427,7 +468,7 @@ def get_config_h_filename(): """Return the path of pyconfig.h.""" if _PYTHON_BUILD: if os.name == "nt": - inc_dir = os.path.dirname(sys._base_executable) + inc_dir = os.path.join(_PROJECT_BASE, 'PC') else: inc_dir = _PROJECT_BASE else: @@ -468,29 +509,44 @@ def get_path(name, scheme=get_default_scheme(), vars=None, expand=True): def _init_config_vars(): global _CONFIG_VARS _CONFIG_VARS = {} + + prefix = os.path.normpath(sys.prefix) + exec_prefix = os.path.normpath(sys.exec_prefix) + base_prefix = _BASE_PREFIX + base_exec_prefix = _BASE_EXEC_PREFIX + + try: + abiflags = sys.abiflags + except AttributeError: + abiflags = '' + + if os.name == 'posix': + _init_posix(_CONFIG_VARS) + # If we are cross-compiling, load the prefixes from the Makefile instead. + if '_PYTHON_PROJECT_BASE' in os.environ: + prefix = _CONFIG_VARS['host_prefix'] + exec_prefix = _CONFIG_VARS['host_exec_prefix'] + base_prefix = _CONFIG_VARS['host_prefix'] + base_exec_prefix = _CONFIG_VARS['host_exec_prefix'] + abiflags = _CONFIG_VARS['ABIFLAGS'] + # Normalized versions of prefix and exec_prefix are handy to have; # in fact, these are the standard versions used most places in the # Distutils. - _PREFIX = os.path.normpath(sys.prefix) - _EXEC_PREFIX = os.path.normpath(sys.exec_prefix) - _CONFIG_VARS['prefix'] = _PREFIX # FIXME: This gets overwriten by _init_posix. - _CONFIG_VARS['exec_prefix'] = _EXEC_PREFIX # FIXME: This gets overwriten by _init_posix. + _CONFIG_VARS['prefix'] = prefix + _CONFIG_VARS['exec_prefix'] = exec_prefix _CONFIG_VARS['py_version'] = _PY_VERSION _CONFIG_VARS['py_version_short'] = _PY_VERSION_SHORT _CONFIG_VARS['py_version_nodot'] = _PY_VERSION_SHORT_NO_DOT - _CONFIG_VARS['installed_base'] = _BASE_PREFIX - _CONFIG_VARS['base'] = _PREFIX - _CONFIG_VARS['installed_platbase'] = _BASE_EXEC_PREFIX - _CONFIG_VARS['platbase'] = _EXEC_PREFIX + _CONFIG_VARS['installed_base'] = base_prefix + _CONFIG_VARS['base'] = prefix + _CONFIG_VARS['installed_platbase'] = base_exec_prefix + _CONFIG_VARS['platbase'] = exec_prefix _CONFIG_VARS['projectbase'] = _PROJECT_BASE _CONFIG_VARS['platlibdir'] = sys.platlibdir _CONFIG_VARS['implementation'] = _get_implementation() _CONFIG_VARS['implementation_lower'] = _get_implementation().lower() - try: - _CONFIG_VARS['abiflags'] = sys.abiflags - except AttributeError: - # sys.abiflags may not be defined on all platforms. - _CONFIG_VARS['abiflags'] = '' + _CONFIG_VARS['abiflags'] = abiflags try: _CONFIG_VARS['py_version_nodot_plat'] = sys.winver.replace('.', '') except AttributeError: @@ -499,8 +555,6 @@ def _init_config_vars(): if os.name == 'nt': _init_non_posix(_CONFIG_VARS) _CONFIG_VARS['VPATH'] = sys._vpath - if os.name == 'posix': - _init_posix(_CONFIG_VARS) if _HAS_USER_BASE: # Setting 'userbase' is done below the call to the # init function to enable using 'get_config_var' in @@ -550,7 +604,16 @@ def get_config_vars(*args): global _CONFIG_VARS_INITIALIZED # Avoid claiming the lock once initialization is complete. - if not _CONFIG_VARS_INITIALIZED: + if _CONFIG_VARS_INITIALIZED: + # GH-126789: If sys.prefix or sys.exec_prefix were updated, invalidate the cache. + prefix = os.path.normpath(sys.prefix) + exec_prefix = os.path.normpath(sys.exec_prefix) + if _CONFIG_VARS['prefix'] != prefix or _CONFIG_VARS['exec_prefix'] != exec_prefix: + with _CONFIG_VARS_LOCK: + _CONFIG_VARS_INITIALIZED = False + _init_config_vars() + else: + # Initialize the config_vars cache. with _CONFIG_VARS_LOCK: # Test again with the lock held to avoid races. Note that # we test _CONFIG_VARS here, not _CONFIG_VARS_INITIALIZED, @@ -558,15 +621,6 @@ def get_config_vars(*args): # don't re-enter init_config_vars(). if _CONFIG_VARS is None: _init_config_vars() - else: - # If the site module initialization happened after _CONFIG_VARS was - # initialized, a virtual environment might have been activated, resulting in - # variables like sys.prefix changing their value, so we need to re-init the - # config vars (see GH-126789). - if _CONFIG_VARS['base'] != os.path.normpath(sys.prefix): - with _CONFIG_VARS_LOCK: - _CONFIG_VARS_INITIALIZED = False - _init_config_vars() if args: vals = [] @@ -627,34 +681,34 @@ def get_platform(): # Set for cross builds explicitly if "_PYTHON_HOST_PLATFORM" in os.environ: - return os.environ["_PYTHON_HOST_PLATFORM"] - - # Try to distinguish various flavours of Unix - osname, host, release, version, machine = os.uname() - - # Convert the OS name to lowercase, remove '/' characters, and translate - # spaces (for "Power Macintosh") - osname = osname.lower().replace('/', '') - machine = machine.replace(' ', '_') - machine = machine.replace('/', '-') - - if osname[:5] == "linux": - if sys.platform == "android": - osname = "android" - release = get_config_var("ANDROID_API_LEVEL") - - # Wheel tags use the ABI names from Android's own tools. - machine = { - "x86_64": "x86_64", - "i686": "x86", - "aarch64": "arm64_v8a", - "armv7l": "armeabi_v7a", - }[machine] - else: - # At least on Linux/Intel, 'machine' is the processor -- - # i386, etc. - # XXX what about Alpha, SPARC, etc? - return f"{osname}-{machine}" + osname, _, machine = os.environ["_PYTHON_HOST_PLATFORM"].partition('-') + release = None + else: + # Try to distinguish various flavours of Unix + osname, host, release, version, machine = os.uname() + + # Convert the OS name to lowercase, remove '/' characters, and translate + # spaces (for "Power Macintosh") + osname = osname.lower().replace('/', '') + machine = machine.replace(' ', '_') + machine = machine.replace('/', '-') + + if osname == "android" or sys.platform == "android": + osname = "android" + release = get_config_var("ANDROID_API_LEVEL") + + # Wheel tags use the ABI names from Android's own tools. + machine = { + "x86_64": "x86_64", + "i686": "x86", + "aarch64": "arm64_v8a", + "armv7l": "armeabi_v7a", + }[machine] + elif osname == "linux": + # At least on Linux/Intel, 'machine' is the processor -- + # i386, etc. + # XXX what about Alpha, SPARC, etc? + return f"{osname}-{machine}" elif osname[:5] == "sunos": if release[0] >= "5": # SunOS 5 == Solaris 2 osname = "solaris" @@ -686,7 +740,7 @@ def get_platform(): get_config_vars(), osname, release, machine) - return f"{osname}-{release}-{machine}" + return '-'.join(map(str, filter(None, (osname, release, machine)))) def get_python_version(): @@ -705,6 +759,15 @@ def expand_makefile_vars(s, vars): variable expansions; if 'vars' is the output of 'parse_makefile()', you're fine. Returns a variable-expanded version of 's'. """ + + import warnings + warnings.warn( + 'sysconfig.expand_makefile_vars is deprecated and will be removed in ' + 'Python 3.16. Use sysconfig.get_paths(vars=...) instead.', + DeprecationWarning, + stacklevel=2, + ) + import re _findvar1_rx = r"\$\(([A-Za-z][A-Za-z0-9_]*)\)" diff --git a/Lib/sysconfig/__main__.py b/Lib/sysconfig/__main__.py index d7257b9d2d0..bc2197cfe79 100644 --- a/Lib/sysconfig/__main__.py +++ b/Lib/sysconfig/__main__.py @@ -1,10 +1,13 @@ +import json import os import sys +import types from sysconfig import ( _ALWAYS_STR, _PYTHON_BUILD, _get_sysconfigdata_name, get_config_h_filename, + get_config_var, get_config_vars, get_default_scheme, get_makefile_filename, @@ -157,6 +160,19 @@ def _print_config_dict(d, stream): print ("}", file=stream) +def _get_pybuilddir(): + pybuilddir = f'build/lib.{get_platform()}-{get_python_version()}' + if get_config_var('Py_DEBUG') == '1': + pybuilddir += '-pydebug' + return pybuilddir + + +def _get_json_data_name(): + name = _get_sysconfigdata_name() + assert name.startswith('_sysconfigdata') + return name.replace('_sysconfigdata', '_sysconfig_vars') + '.json' + + def _generate_posix_vars(): """Generate the Python module containing build-time variables.""" vars = {} @@ -185,6 +201,8 @@ def _generate_posix_vars(): if _PYTHON_BUILD: vars['BLDSHARED'] = vars['LDSHARED'] + name = _get_sysconfigdata_name() + # There's a chicken-and-egg situation on OS X with regards to the # _sysconfigdata module after the changes introduced by #15298: # get_config_vars() is called by get_platform() as part of the @@ -196,16 +214,13 @@ def _generate_posix_vars(): # _sysconfigdata module manually and populate it with the build vars. # This is more than sufficient for ensuring the subsequent call to # get_platform() succeeds. - name = _get_sysconfigdata_name() - if 'darwin' in sys.platform: - import types - module = types.ModuleType(name) - module.build_time_vars = vars - sys.modules[name] = module + # GH-127178: Since we started generating a .json file, we also need this to + # be able to run sysconfig.get_config_vars(). + module = types.ModuleType(name) + module.build_time_vars = vars + sys.modules[name] = module - pybuilddir = f'build/lib.{get_platform()}-{get_python_version()}' - if hasattr(sys, "gettotalrefcount"): - pybuilddir += '-pydebug' + pybuilddir = _get_pybuilddir() os.makedirs(pybuilddir, exist_ok=True) destfile = os.path.join(pybuilddir, name + '.py') @@ -215,6 +230,19 @@ def _generate_posix_vars(): f.write('build_time_vars = ') _print_config_dict(vars, stream=f) + print(f'Written {destfile}') + + install_vars = get_config_vars() + # Fix config vars to match the values after install (of the default environment) + install_vars['projectbase'] = install_vars['BINDIR'] + install_vars['srcdir'] = install_vars['LIBPL'] + # Write a JSON file with the output of sysconfig.get_config_vars + jsonfile = os.path.join(pybuilddir, _get_json_data_name()) + with open(jsonfile, 'w') as f: + json.dump(install_vars, f, indent=2) + + print(f'Written {jsonfile}') + # Create file used for sys.path fixup -- see Modules/getpath.c with open('pybuilddir.txt', 'w', encoding='utf8') as f: f.write(pybuilddir) diff --git a/Lib/tabnanny.py b/Lib/tabnanny.py old mode 100755 new mode 100644 index d06c4c221e9..c0097351b26 --- a/Lib/tabnanny.py +++ b/Lib/tabnanny.py @@ -1,5 +1,3 @@ -#! /usr/bin/env python3 - """The Tab Nanny despises ambiguous indentation. She knows no mercy. tabnanny -- Detection of ambiguous indentation diff --git a/Lib/tarfile.py b/Lib/tarfile.py old mode 100755 new mode 100644 index 04fda115971..c7e9f7d681a --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 #------------------------------------------------------------------- # tarfile.py #------------------------------------------------------------------- @@ -46,7 +45,6 @@ import struct import copy import re -import warnings try: import pwd @@ -69,7 +67,7 @@ "DEFAULT_FORMAT", "open","fully_trusted_filter", "data_filter", "tar_filter", "FilterError", "AbsoluteLinkError", "OutsideDestinationError", "SpecialFileError", "AbsolutePathError", - "LinkOutsideDestinationError"] + "LinkOutsideDestinationError", "LinkFallbackError"] #--------------------------------------------------------- @@ -341,7 +339,7 @@ class _Stream: """ def __init__(self, name, mode, comptype, fileobj, bufsize, - compresslevel): + compresslevel, preset): """Construct a _Stream object. """ self._extfileobj = True @@ -355,7 +353,7 @@ def __init__(self, name, mode, comptype, fileobj, bufsize, fileobj = _StreamProxy(fileobj) comptype = fileobj.getcomptype() - self.name = name or "" + self.name = os.fspath(name) if name is not None else "" self.mode = mode self.comptype = comptype self.fileobj = fileobj @@ -395,17 +393,23 @@ def __init__(self, name, mode, comptype, fileobj, bufsize, import lzma except ImportError: raise CompressionError("lzma module is not available") from None - - # XXX: RUSTPYTHON; xz is not supported yet - raise CompressionError("lzma module is not available") from None - if mode == "r": self.dbuf = b"" self.cmp = lzma.LZMADecompressor() self.exception = lzma.LZMAError else: - self.cmp = lzma.LZMACompressor() - + self.cmp = lzma.LZMACompressor(preset=preset) + elif comptype == "zst": + try: + from compression import zstd + except ImportError: + raise CompressionError("compression.zstd module is not available") from None + if mode == "r": + self.dbuf = b"" + self.cmp = zstd.ZstdDecompressor() + self.exception = zstd.ZstdError + else: + self.cmp = zstd.ZstdCompressor() elif comptype != "tar": raise CompressionError("unknown compression type %r" % comptype) @@ -597,6 +601,8 @@ def getcomptype(self): return "bz2" elif self.buf.startswith((b"\x5d\x00\x00\x80", b"\xfd7zXZ")): return "xz" + elif self.buf.startswith(b"\x28\xb5\x2f\xfd"): + return "zst" else: return "tar" @@ -641,6 +647,10 @@ def __init__(self, fileobj, offset, size, name, blockinfo=None): def flush(self): pass + @property + def mode(self): + return 'rb' + def readable(self): return True @@ -756,10 +766,22 @@ def __init__(self, tarinfo, path): super().__init__(f'{tarinfo.name!r} would link to {path!r}, ' + 'which is outside the destination') +class LinkFallbackError(FilterError): + def __init__(self, tarinfo, path): + self.tarinfo = tarinfo + self._path = path + super().__init__(f'link {tarinfo.name!r} would be extracted as a ' + + f'copy of {path!r}, which was rejected') + +# Errors caused by filters -- both "fatal" and "non-fatal" -- that +# we consider to be issues with the argument, rather than a bug in the +# filter function +_FILTER_ERRORS = (FilterError, OSError, ExtractError) + def _get_filtered_attrs(member, dest_path, for_data=True): new_attrs = {} name = member.name - dest_path = os.path.realpath(dest_path) + dest_path = os.path.realpath(dest_path, strict=os.path.ALLOW_MISSING) # Strip leading / (tar's directory separator) from filenames. # Include os.sep (target OS directory separator) as well. if name.startswith(('/', os.sep)): @@ -769,7 +791,8 @@ def _get_filtered_attrs(member, dest_path, for_data=True): # For example, 'C:/foo' on Windows. raise AbsolutePathError(member) # Ensure we stay in the destination - target_path = os.path.realpath(os.path.join(dest_path, name)) + target_path = os.path.realpath(os.path.join(dest_path, name), + strict=os.path.ALLOW_MISSING) if os.path.commonpath([target_path, dest_path]) != dest_path: raise OutsideDestinationError(member, target_path) # Limit permissions (no high bits, and go-w) @@ -807,6 +830,9 @@ def _get_filtered_attrs(member, dest_path, for_data=True): if member.islnk() or member.issym(): if os.path.isabs(member.linkname): raise AbsoluteLinkError(member) + normalized = os.path.normpath(member.linkname) + if normalized != member.linkname: + new_attrs['linkname'] = normalized if member.issym(): target_path = os.path.join(dest_path, os.path.dirname(name), @@ -814,7 +840,8 @@ def _get_filtered_attrs(member, dest_path, for_data=True): else: target_path = os.path.join(dest_path, member.linkname) - target_path = os.path.realpath(target_path) + target_path = os.path.realpath(target_path, + strict=os.path.ALLOW_MISSING) if os.path.commonpath([target_path, dest_path]) != dest_path: raise LinkOutsideDestinationError(member, target_path) return new_attrs @@ -847,6 +874,9 @@ def data_filter(member, dest_path): # Sentinel for replace() defaults, meaning "don't change the attribute" _KEEP = object() +# Header length is digits followed by a space. +_header_length_prefix_re = re.compile(br"([0-9]{1,20}) ") + class TarInfo(object): """Informational class which holds the details about an archive member given by a tar header block. @@ -877,7 +907,7 @@ class TarInfo(object): pax_headers = ('A dictionary containing key-value pairs of an ' 'associated pax extended header.'), sparse = 'Sparse member information.', - tarfile = None, + _tarfile = None, _sparse_structs = None, _link_target = None, ) @@ -906,6 +936,24 @@ def __init__(self, name=""): self.sparse = None # sparse member information self.pax_headers = {} # pax header information + @property + def tarfile(self): + import warnings + warnings.warn( + 'The undocumented "tarfile" attribute of TarInfo objects ' + + 'is deprecated and will be removed in Python 3.16', + DeprecationWarning, stacklevel=2) + return self._tarfile + + @tarfile.setter + def tarfile(self, tarfile): + import warnings + warnings.warn( + 'The undocumented "tarfile" attribute of TarInfo objects ' + + 'is deprecated and will be removed in Python 3.16', + DeprecationWarning, stacklevel=2) + self._tarfile = tarfile + @property def path(self): 'In pax headers, "name" is called "path".' @@ -1200,7 +1248,7 @@ def _create_pax_generic_header(cls, pax_headers, type, encoding): for keyword, value in pax_headers.items(): keyword = keyword.encode("utf-8") if binary: - # Try to restore the original byte representation of `value'. + # Try to restore the original byte representation of 'value'. # Needless to say, that the encoding must match the string. value = value.encode(encoding, "surrogateescape") else: @@ -1416,37 +1464,59 @@ def _proc_pax(self, tarfile): else: pax_headers = tarfile.pax_headers.copy() - # Check if the pax header contains a hdrcharset field. This tells us - # the encoding of the path, linkpath, uname and gname fields. Normally, - # these fields are UTF-8 encoded but since POSIX.1-2008 tar - # implementations are allowed to store them as raw binary strings if - # the translation to UTF-8 fails. - match = re.search(br"\d+ hdrcharset=([^\n]+)\n", buf) - if match is not None: - pax_headers["hdrcharset"] = match.group(1).decode("utf-8") - - # For the time being, we don't care about anything other than "BINARY". - # The only other value that is currently allowed by the standard is - # "ISO-IR 10646 2000 UTF-8" in other words UTF-8. - hdrcharset = pax_headers.get("hdrcharset") - if hdrcharset == "BINARY": - encoding = tarfile.encoding - else: - encoding = "utf-8" - # Parse pax header information. A record looks like that: # "%d %s=%s\n" % (length, keyword, value). length is the size # of the complete record including the length field itself and - # the newline. keyword and value are both UTF-8 encoded strings. - regex = re.compile(br"(\d+) ([^=]+)=") + # the newline. pos = 0 - while match := regex.match(buf, pos): - length, keyword = match.groups() - length = int(length) - if length == 0: + encoding = None + raw_headers = [] + while len(buf) > pos and buf[pos] != 0x00: + if not (match := _header_length_prefix_re.match(buf, pos)): raise InvalidHeaderError("invalid header") - value = buf[match.end(2) + 1:match.start(1) + length - 1] + try: + length = int(match.group(1)) + except ValueError: + raise InvalidHeaderError("invalid header") + # Headers must be at least 5 bytes, shortest being '5 x=\n'. + # Value is allowed to be empty. + if length < 5: + raise InvalidHeaderError("invalid header") + if pos + length > len(buf): + raise InvalidHeaderError("invalid header") + + header_value_end_offset = match.start(1) + length - 1 # Last byte of the header + keyword_and_value = buf[match.end(1) + 1:header_value_end_offset] + raw_keyword, equals, raw_value = keyword_and_value.partition(b"=") + # Check the framing of the header. The last character must be '\n' (0x0A) + if not raw_keyword or equals != b"=" or buf[header_value_end_offset] != 0x0A: + raise InvalidHeaderError("invalid header") + raw_headers.append((length, raw_keyword, raw_value)) + + # Check if the pax header contains a hdrcharset field. This tells us + # the encoding of the path, linkpath, uname and gname fields. Normally, + # these fields are UTF-8 encoded but since POSIX.1-2008 tar + # implementations are allowed to store them as raw binary strings if + # the translation to UTF-8 fails. For the time being, we don't care about + # anything other than "BINARY". The only other value that is currently + # allowed by the standard is "ISO-IR 10646 2000 UTF-8" in other words UTF-8. + # Note that we only follow the initial 'hdrcharset' setting to preserve + # the initial behavior of the 'tarfile' module. + if raw_keyword == b"hdrcharset" and encoding is None: + if raw_value == b"BINARY": + encoding = tarfile.encoding + else: # This branch ensures only the first 'hdrcharset' header is used. + encoding = "utf-8" + + pos += length + + # If no explicit hdrcharset is set, we use UTF-8 as a default. + if encoding is None: + encoding = "utf-8" + + # After parsing the raw headers we can decode them to text. + for length, raw_keyword, raw_value in raw_headers: # Normally, we could just use "utf-8" as the encoding and "strict" # as the error handler, but we better not take the risk. For # example, GNU tar <= 1.23 is known to store filenames it cannot @@ -1454,17 +1524,16 @@ def _proc_pax(self, tarfile): # hdrcharset=BINARY header). # We first try the strict standard encoding, and if that fails we # fall back on the user's encoding and error handler. - keyword = self._decode_pax_field(keyword, "utf-8", "utf-8", + keyword = self._decode_pax_field(raw_keyword, "utf-8", "utf-8", tarfile.errors) if keyword in PAX_NAME_FIELDS: - value = self._decode_pax_field(value, encoding, tarfile.encoding, + value = self._decode_pax_field(raw_value, encoding, tarfile.encoding, tarfile.errors) else: - value = self._decode_pax_field(value, "utf-8", "utf-8", + value = self._decode_pax_field(raw_value, "utf-8", "utf-8", tarfile.errors) pax_headers[keyword] = value - pos += length # Fetch the next header. try: @@ -1479,7 +1548,7 @@ def _proc_pax(self, tarfile): elif "GNU.sparse.size" in pax_headers: # GNU extended sparse format version 0.0. - self._proc_gnusparse_00(next, pax_headers, buf) + self._proc_gnusparse_00(next, raw_headers) elif pax_headers.get("GNU.sparse.major") == "1" and pax_headers.get("GNU.sparse.minor") == "0": # GNU extended sparse format version 1.0. @@ -1501,15 +1570,24 @@ def _proc_pax(self, tarfile): return next - def _proc_gnusparse_00(self, next, pax_headers, buf): + def _proc_gnusparse_00(self, next, raw_headers): """Process a GNU tar extended sparse header, version 0.0. """ offsets = [] - for match in re.finditer(br"\d+ GNU.sparse.offset=(\d+)\n", buf): - offsets.append(int(match.group(1))) numbytes = [] - for match in re.finditer(br"\d+ GNU.sparse.numbytes=(\d+)\n", buf): - numbytes.append(int(match.group(1))) + for _, keyword, value in raw_headers: + if keyword == b"GNU.sparse.offset": + try: + offsets.append(int(value.decode())) + except ValueError: + raise InvalidHeaderError("invalid header") + + elif keyword == b"GNU.sparse.numbytes": + try: + numbytes.append(int(value.decode())) + except ValueError: + raise InvalidHeaderError("invalid header") + next.sparse = list(zip(offsets, numbytes)) def _proc_gnusparse_01(self, next, pax_headers): @@ -1569,6 +1647,9 @@ def _block(self, count): """Round up a byte count by BLOCKSIZE and return it, e.g. _block(834) => 1024. """ + # Only non-negative offsets are allowed + if count < 0: + raise InvalidHeaderError("invalid offset") blocks, remainder = divmod(count, BLOCKSIZE) if remainder: blocks += 1 @@ -1645,14 +1726,14 @@ class TarFile(object): def __init__(self, name=None, mode="r", fileobj=None, format=None, tarinfo=None, dereference=None, ignore_zeros=None, encoding=None, errors="surrogateescape", pax_headers=None, debug=None, - errorlevel=None, copybufsize=None): - """Open an (uncompressed) tar archive `name'. `mode' is either 'r' to + errorlevel=None, copybufsize=None, stream=False): + """Open an (uncompressed) tar archive 'name'. 'mode' is either 'r' to read from an existing archive, 'a' to append data to an existing - file or 'w' to create a new file overwriting an existing one. `mode' + file or 'w' to create a new file overwriting an existing one. 'mode' defaults to 'r'. - If `fileobj' is given, it is used for reading or writing data. If it - can be determined, `mode' is overridden by `fileobj's mode. - `fileobj' is not closed, when TarFile is closed. + If 'fileobj' is given, it is used for reading or writing data. If it + can be determined, 'mode' is overridden by 'fileobj's mode. + 'fileobj' is not closed, when TarFile is closed. """ modes = {"r": "rb", "a": "r+b", "w": "wb", "x": "xb"} if mode not in modes: @@ -1677,6 +1758,8 @@ def __init__(self, name=None, mode="r", fileobj=None, format=None, self.name = os.path.abspath(name) if name else None self.fileobj = fileobj + self.stream = stream + # Init attributes. if format is not None: self.format = format @@ -1709,6 +1792,8 @@ def __init__(self, name=None, mode="r", fileobj=None, format=None, # current position in the archive file self.inodes = {} # dictionary caching the inodes of # archive members already added + self._unames = {} # Cached mappings of uid -> uname + self._gnames = {} # Cached mappings of gid -> gname try: if self.mode == "r": @@ -1764,11 +1849,13 @@ def open(cls, name=None, mode="r", fileobj=None, bufsize=RECORDSIZE, **kwargs): 'r:gz' open for reading with gzip compression 'r:bz2' open for reading with bzip2 compression 'r:xz' open for reading with lzma compression + 'r:zst' open for reading with zstd compression 'a' or 'a:' open for appending, creating the file if necessary 'w' or 'w:' open for writing without compression 'w:gz' open for writing with gzip compression 'w:bz2' open for writing with bzip2 compression 'w:xz' open for writing with lzma compression + 'w:zst' open for writing with zstd compression 'x' or 'x:' create a tarfile exclusively without compression, raise an exception if the file is already created @@ -1778,16 +1865,20 @@ def open(cls, name=None, mode="r", fileobj=None, bufsize=RECORDSIZE, **kwargs): if the file is already created 'x:xz' create an lzma compressed tarfile, raise an exception if the file is already created + 'x:zst' create a zstd compressed tarfile, raise an exception + if the file is already created 'r|*' open a stream of tar blocks with transparent compression 'r|' open an uncompressed stream of tar blocks for reading 'r|gz' open a gzip compressed stream of tar blocks 'r|bz2' open a bzip2 compressed stream of tar blocks 'r|xz' open an lzma compressed stream of tar blocks + 'r|zst' open a zstd compressed stream of tar blocks 'w|' open an uncompressed stream for writing 'w|gz' open a gzip compressed stream for writing 'w|bz2' open a bzip2 compressed stream for writing 'w|xz' open an lzma compressed stream for writing + 'w|zst' open a zstd compressed stream for writing """ if not name and not fileobj: @@ -1832,10 +1923,17 @@ def not_compressed(comptype): if filemode not in ("r", "w"): raise ValueError("mode must be 'r' or 'w'") + if "compresslevel" in kwargs and comptype not in ("gz", "bz2"): + raise ValueError( + "compresslevel is only valid for w|gz and w|bz2 modes" + ) + if "preset" in kwargs and comptype not in ("xz",): + raise ValueError("preset is only valid for w|xz mode") compresslevel = kwargs.pop("compresslevel", 9) + preset = kwargs.pop("preset", None) stream = _Stream(name, filemode, comptype, fileobj, bufsize, - compresslevel) + compresslevel, preset) try: t = cls(name, filemode, stream, **kwargs) except: @@ -1931,9 +2029,6 @@ def xzopen(cls, name, mode="r", fileobj=None, preset=None, **kwargs): except ImportError: raise CompressionError("lzma module is not available") from None - # XXX: RUSTPYTHON; xz is not supported yet - raise CompressionError("lzma module is not available") from None - fileobj = LZMAFile(fileobj or name, mode, preset=preset) try: @@ -1949,12 +2044,48 @@ def xzopen(cls, name, mode="r", fileobj=None, preset=None, **kwargs): t._extfileobj = False return t + @classmethod + def zstopen(cls, name, mode="r", fileobj=None, level=None, options=None, + zstd_dict=None, **kwargs): + """Open zstd compressed tar archive name for reading or writing. + Appending is not allowed. + """ + if mode not in ("r", "w", "x"): + raise ValueError("mode must be 'r', 'w' or 'x'") + + try: + from compression.zstd import ZstdFile, ZstdError + except ImportError: + raise CompressionError("compression.zstd module is not available") from None + + fileobj = ZstdFile( + fileobj or name, + mode, + level=level, + options=options, + zstd_dict=zstd_dict + ) + + try: + t = cls.taropen(name, mode, fileobj, **kwargs) + except (ZstdError, EOFError) as e: + fileobj.close() + if mode == 'r': + raise ReadError("not a zstd file") from e + raise + except Exception: + fileobj.close() + raise + t._extfileobj = False + return t + # All *open() methods are registered here. OPEN_METH = { "tar": "taropen", # uncompressed tar "gz": "gzopen", # gzip compressed tar "bz2": "bz2open", # bzip2 compressed tar - "xz": "xzopen" # lzma compressed tar + "xz": "xzopen", # lzma compressed tar + "zst": "zstopen", # zstd compressed tar } #-------------------------------------------------------------------------- @@ -1982,7 +2113,7 @@ def close(self): self.fileobj.close() def getmember(self, name): - """Return a TarInfo object for member `name'. If `name' can not be + """Return a TarInfo object for member 'name'. If 'name' can not be found in the archive, KeyError is raised. If a member occurs more than once in the archive, its last occurrence is assumed to be the most up-to-date version. @@ -2010,9 +2141,9 @@ def getnames(self): def gettarinfo(self, name=None, arcname=None, fileobj=None): """Create a TarInfo object from the result of os.stat or equivalent - on an existing file. The file is either named by `name', or - specified as a file object `fileobj' with a file descriptor. If - given, `arcname' specifies an alternative name for the file in the + on an existing file. The file is either named by 'name', or + specified as a file object 'fileobj' with a file descriptor. If + given, 'arcname' specifies an alternative name for the file in the archive, otherwise, the name is taken from the 'name' attribute of 'fileobj', or the 'name' argument. The name should be a text string. @@ -2036,7 +2167,7 @@ def gettarinfo(self, name=None, arcname=None, fileobj=None): # Now, fill the TarInfo object with # information specific for the file. tarinfo = self.tarinfo() - tarinfo.tarfile = self # Not needed + tarinfo._tarfile = self # To be removed in 3.16. # Use os.stat or os.lstat, depending on if symlinks shall be resolved. if fileobj is None: @@ -2090,16 +2221,23 @@ def gettarinfo(self, name=None, arcname=None, fileobj=None): tarinfo.mtime = statres.st_mtime tarinfo.type = type tarinfo.linkname = linkname + + # Calls to pwd.getpwuid() and grp.getgrgid() tend to be expensive. To + # speed things up, cache the resolved usernames and group names. if pwd: - try: - tarinfo.uname = pwd.getpwuid(tarinfo.uid)[0] - except KeyError: - pass + if tarinfo.uid not in self._unames: + try: + self._unames[tarinfo.uid] = pwd.getpwuid(tarinfo.uid)[0] + except KeyError: + self._unames[tarinfo.uid] = '' + tarinfo.uname = self._unames[tarinfo.uid] if grp: - try: - tarinfo.gname = grp.getgrgid(tarinfo.gid)[0] - except KeyError: - pass + if tarinfo.gid not in self._gnames: + try: + self._gnames[tarinfo.gid] = grp.getgrgid(tarinfo.gid)[0] + except KeyError: + self._gnames[tarinfo.gid] = '' + tarinfo.gname = self._gnames[tarinfo.gid] if type in (CHRTYPE, BLKTYPE): if hasattr(os, "major") and hasattr(os, "minor"): @@ -2108,11 +2246,15 @@ def gettarinfo(self, name=None, arcname=None, fileobj=None): return tarinfo def list(self, verbose=True, *, members=None): - """Print a table of contents to sys.stdout. If `verbose' is False, only - the names of the members are printed. If it is True, an `ls -l'-like - output is produced. `members' is optional and must be a subset of the + """Print a table of contents to sys.stdout. If 'verbose' is False, only + the names of the members are printed. If it is True, an 'ls -l'-like + output is produced. 'members' is optional and must be a subset of the list returned by getmembers(). """ + # Convert tarinfo type to stat type. + type2mode = {REGTYPE: stat.S_IFREG, SYMTYPE: stat.S_IFLNK, + FIFOTYPE: stat.S_IFIFO, CHRTYPE: stat.S_IFCHR, + DIRTYPE: stat.S_IFDIR, BLKTYPE: stat.S_IFBLK} self._check() if members is None: @@ -2122,7 +2264,8 @@ def list(self, verbose=True, *, members=None): if tarinfo.mode is None: _safe_print("??????????") else: - _safe_print(stat.filemode(tarinfo.mode)) + modetype = type2mode.get(tarinfo.type, 0) + _safe_print(stat.filemode(modetype | tarinfo.mode)) _safe_print("%s/%s" % (tarinfo.uname or tarinfo.uid, tarinfo.gname or tarinfo.gid)) if tarinfo.ischr() or tarinfo.isblk(): @@ -2146,11 +2289,11 @@ def list(self, verbose=True, *, members=None): print() def add(self, name, arcname=None, recursive=True, *, filter=None): - """Add the file `name' to the archive. `name' may be any type of file - (directory, fifo, symbolic link, etc.). If given, `arcname' + """Add the file 'name' to the archive. 'name' may be any type of file + (directory, fifo, symbolic link, etc.). If given, 'arcname' specifies an alternative name for the file in the archive. Directories are added recursively by default. This can be avoided by - setting `recursive' to False. `filter' is a function + setting 'recursive' to False. 'filter' is a function that expects a TarInfo object argument and returns the changed TarInfo object, if it returns None the TarInfo object will be excluded from the archive. @@ -2197,13 +2340,16 @@ def add(self, name, arcname=None, recursive=True, *, filter=None): self.addfile(tarinfo) def addfile(self, tarinfo, fileobj=None): - """Add the TarInfo object `tarinfo' to the archive. If `fileobj' is - given, it should be a binary file, and tarinfo.size bytes are read - from it and added to the archive. You can create TarInfo objects - directly, or by using gettarinfo(). + """Add the TarInfo object 'tarinfo' to the archive. If 'tarinfo' represents + a non zero-size regular file, the 'fileobj' argument should be a binary file, + and tarinfo.size bytes are read from it and added to the archive. + You can create TarInfo objects directly, or by using gettarinfo(). """ self._check("awx") + if fileobj is None and tarinfo.isreg() and tarinfo.size != 0: + raise ValueError("fileobj not provided for non zero-size regular file") + tarinfo = copy.copy(tarinfo) buf = tarinfo.tobuf(self.format, self.encoding, self.errors) @@ -2225,12 +2371,7 @@ def _get_filter_function(self, filter): if filter is None: filter = self.extraction_filter if filter is None: - warnings.warn( - 'Python 3.14 will, by default, filter extracted tar ' - + 'archives and reject files or modify their metadata. ' - + 'Use the filter argument to control this behavior.', - DeprecationWarning) - return fully_trusted_filter + return data_filter if isinstance(filter, str): raise TypeError( 'String names are not supported for ' @@ -2248,12 +2389,12 @@ def extractall(self, path=".", members=None, *, numeric_owner=False, filter=None): """Extract all members from the archive to the current working directory and set owner, modification time and permissions on - directories afterwards. `path' specifies a different directory - to extract to. `members' is optional and must be a subset of the - list returned by getmembers(). If `numeric_owner` is True, only + directories afterwards. 'path' specifies a different directory + to extract to. 'members' is optional and must be a subset of the + list returned by getmembers(). If 'numeric_owner' is True, only the numbers for user/group names are used and not the names. - The `filter` function will be called on each member just + The 'filter' function will be called on each member just before extraction. It can return a changed TarInfo or None to skip the member. String names of common filters are accepted. @@ -2265,81 +2406,124 @@ def extractall(self, path=".", members=None, *, numeric_owner=False, members = self for member in members: - tarinfo = self._get_extract_tarinfo(member, filter_function, path) + tarinfo, unfiltered = self._get_extract_tarinfo( + member, filter_function, path) if tarinfo is None: continue if tarinfo.isdir(): # For directories, delay setting attributes until later, # since permissions can interfere with extraction and # extracting contents can reset mtime. - directories.append(tarinfo) + directories.append(unfiltered) self._extract_one(tarinfo, path, set_attrs=not tarinfo.isdir(), - numeric_owner=numeric_owner) + numeric_owner=numeric_owner, + filter_function=filter_function) # Reverse sort directories. directories.sort(key=lambda a: a.name, reverse=True) + # Set correct owner, mtime and filemode on directories. - for tarinfo in directories: - dirpath = os.path.join(path, tarinfo.name) + for unfiltered in directories: try: + # Need to re-apply any filter, to take the *current* filesystem + # state into account. + try: + tarinfo = filter_function(unfiltered, path) + except _FILTER_ERRORS as exc: + self._log_no_directory_fixup(unfiltered, repr(exc)) + continue + if tarinfo is None: + self._log_no_directory_fixup(unfiltered, + 'excluded by filter') + continue + dirpath = os.path.join(path, tarinfo.name) + try: + lstat = os.lstat(dirpath) + except FileNotFoundError: + self._log_no_directory_fixup(tarinfo, 'missing') + continue + if not stat.S_ISDIR(lstat.st_mode): + # This is no longer a directory; presumably a later + # member overwrote the entry. + self._log_no_directory_fixup(tarinfo, 'not a directory') + continue self.chown(tarinfo, dirpath, numeric_owner=numeric_owner) self.utime(tarinfo, dirpath) self.chmod(tarinfo, dirpath) except ExtractError as e: self._handle_nonfatal_error(e) + def _log_no_directory_fixup(self, member, reason): + self._dbg(2, "tarfile: Not fixing up directory %r (%s)" % + (member.name, reason)) + def extract(self, member, path="", set_attrs=True, *, numeric_owner=False, filter=None): """Extract a member from the archive to the current working directory, using its full name. Its file information is extracted as accurately - as possible. `member' may be a filename or a TarInfo object. You can - specify a different directory using `path'. File attributes (owner, - mtime, mode) are set unless `set_attrs' is False. If `numeric_owner` + as possible. 'member' may be a filename or a TarInfo object. You can + specify a different directory using 'path'. File attributes (owner, + mtime, mode) are set unless 'set_attrs' is False. If 'numeric_owner' is True, only the numbers for user/group names are used and not the names. - The `filter` function will be called before extraction. + The 'filter' function will be called before extraction. It can return a changed TarInfo or None to skip the member. String names of common filters are accepted. """ filter_function = self._get_filter_function(filter) - tarinfo = self._get_extract_tarinfo(member, filter_function, path) + tarinfo, unfiltered = self._get_extract_tarinfo( + member, filter_function, path) if tarinfo is not None: self._extract_one(tarinfo, path, set_attrs, numeric_owner) def _get_extract_tarinfo(self, member, filter_function, path): - """Get filtered TarInfo (or None) from member, which might be a str""" + """Get (filtered, unfiltered) TarInfos from *member* + + *member* might be a string. + + Return (None, None) if not found. + """ + if isinstance(member, str): - tarinfo = self.getmember(member) + unfiltered = self.getmember(member) else: - tarinfo = member + unfiltered = member - unfiltered = tarinfo + filtered = None try: - tarinfo = filter_function(tarinfo, path) - except (OSError, FilterError) as e: + filtered = filter_function(unfiltered, path) + except (OSError, UnicodeEncodeError, FilterError) as e: self._handle_fatal_error(e) except ExtractError as e: self._handle_nonfatal_error(e) - if tarinfo is None: + if filtered is None: self._dbg(2, "tarfile: Excluded %r" % unfiltered.name) - return None + return None, None + # Prepare the link target for makelink(). - if tarinfo.islnk(): - tarinfo = copy.copy(tarinfo) - tarinfo._link_target = os.path.join(path, tarinfo.linkname) - return tarinfo + if filtered.islnk(): + filtered = copy.copy(filtered) + filtered._link_target = os.path.join(path, filtered.linkname) + return filtered, unfiltered - def _extract_one(self, tarinfo, path, set_attrs, numeric_owner): - """Extract from filtered tarinfo to disk""" + def _extract_one(self, tarinfo, path, set_attrs, numeric_owner, + filter_function=None): + """Extract from filtered tarinfo to disk. + + filter_function is only used when extracting a *different* + member (e.g. as fallback to creating a symlink) + """ self._check("r") try: self._extract_member(tarinfo, os.path.join(path, tarinfo.name), set_attrs=set_attrs, - numeric_owner=numeric_owner) - except OSError as e: + numeric_owner=numeric_owner, + filter_function=filter_function, + extraction_root=path) + except (OSError, UnicodeEncodeError) as e: self._handle_fatal_error(e) except ExtractError as e: self._handle_nonfatal_error(e) @@ -2364,10 +2548,10 @@ def _handle_fatal_error(self, e): self._dbg(1, "tarfile: %s %s" % (type(e).__name__, e)) def extractfile(self, member): - """Extract a member from the archive as a file object. `member' may be - a filename or a TarInfo object. If `member' is a regular file or + """Extract a member from the archive as a file object. 'member' may be + a filename or a TarInfo object. If 'member' is a regular file or a link, an io.BufferedReader object is returned. For all other - existing members, None is returned. If `member' does not appear + existing members, None is returned. If 'member' does not appear in the archive, KeyError is raised. """ self._check("r") @@ -2396,9 +2580,13 @@ def extractfile(self, member): return None def _extract_member(self, tarinfo, targetpath, set_attrs=True, - numeric_owner=False): - """Extract the TarInfo object tarinfo to a physical + numeric_owner=False, *, filter_function=None, + extraction_root=None): + """Extract the filtered TarInfo object tarinfo to a physical file called targetpath. + + filter_function is only used when extracting a *different* + member (e.g. as fallback to creating a symlink) """ # Fetch the TarInfo object for the given name # and build the destination pathname, replacing @@ -2411,7 +2599,7 @@ def _extract_member(self, tarinfo, targetpath, set_attrs=True, if upperdirs and not os.path.exists(upperdirs): # Create directories that are not part of the archive with # default permissions. - os.makedirs(upperdirs) + os.makedirs(upperdirs, exist_ok=True) if tarinfo.islnk() or tarinfo.issym(): self._dbg(1, "%s -> %s" % (tarinfo.name, tarinfo.linkname)) @@ -2427,7 +2615,10 @@ def _extract_member(self, tarinfo, targetpath, set_attrs=True, elif tarinfo.ischr() or tarinfo.isblk(): self.makedev(tarinfo, targetpath) elif tarinfo.islnk() or tarinfo.issym(): - self.makelink(tarinfo, targetpath) + self.makelink_with_filter( + tarinfo, targetpath, + filter_function=filter_function, + extraction_root=extraction_root) elif tarinfo.type not in SUPPORTED_TYPES: self.makeunknown(tarinfo, targetpath) else: @@ -2510,10 +2701,18 @@ def makedev(self, tarinfo, targetpath): os.makedev(tarinfo.devmajor, tarinfo.devminor)) def makelink(self, tarinfo, targetpath): + return self.makelink_with_filter(tarinfo, targetpath, None, None) + + def makelink_with_filter(self, tarinfo, targetpath, + filter_function, extraction_root): """Make a (symbolic) link called targetpath. If it cannot be created (platform limitation), we try to make a copy of the referenced file instead of a link. + + filter_function is only used when extracting a *different* + member (e.g. as fallback to creating a link). """ + keyerror_to_extracterror = False try: # For systems that support symbolic and hard links. if tarinfo.issym(): @@ -2521,18 +2720,41 @@ def makelink(self, tarinfo, targetpath): # Avoid FileExistsError on following os.symlink. os.unlink(targetpath) os.symlink(tarinfo.linkname, targetpath) + return else: if os.path.exists(tarinfo._link_target): + if os.path.lexists(targetpath): + # Avoid FileExistsError on following os.link. + os.unlink(targetpath) os.link(tarinfo._link_target, targetpath) - else: - self._extract_member(self._find_link_target(tarinfo), - targetpath) + return except symlink_exception: + keyerror_to_extracterror = True + + try: + unfiltered = self._find_link_target(tarinfo) + except KeyError: + if keyerror_to_extracterror: + raise ExtractError( + "unable to resolve link inside archive") from None + else: + raise + + if filter_function is None: + filtered = unfiltered + else: + if extraction_root is None: + raise ExtractError( + "makelink_with_filter: if filter_function is not None, " + + "extraction_root must also not be None") try: - self._extract_member(self._find_link_target(tarinfo), - targetpath) - except KeyError: - raise ExtractError("unable to resolve link inside archive") from None + filtered = filter_function(unfiltered, extraction_root) + except _FILTER_ERRORS as cause: + raise LinkFallbackError(tarinfo, unfiltered.name) from cause + if filtered is not None: + self._extract_member(filtered, targetpath, + filter_function=filter_function, + extraction_root=extraction_root) def chown(self, tarinfo, targetpath, numeric_owner): """Set owner of targetpath according to tarinfo. If numeric_owner @@ -2564,7 +2786,8 @@ def chown(self, tarinfo, targetpath, numeric_owner): os.lchown(targetpath, u, g) else: os.chown(targetpath, u, g) - except OSError as e: + except (OSError, OverflowError) as e: + # OverflowError can be raised if an ID doesn't fit in 'id_t' raise ExtractError("could not change owner") from e def chmod(self, tarinfo, targetpath): @@ -2647,7 +2870,9 @@ def next(self): break if tarinfo is not None: - self.members.append(tarinfo) + # if streaming the file we do not want to cache the tarinfo + if not self.stream: + self.members.append(tarinfo) else: self._loaded = True @@ -2698,11 +2923,12 @@ def _getmember(self, name, tarinfo=None, normalize=False): def _load(self): """Read through the entire archive file and look for readable - members. + members. This should not run if the file is set to stream. """ - while self.next() is not None: - pass - self._loaded = True + if not self.stream: + while self.next() is not None: + pass + self._loaded = True def _check(self, mode=None): """Check if TarFile is still open, and if the operation's mode @@ -2812,7 +3038,7 @@ def main(): import argparse description = 'A simple command-line interface for tarfile module.' - parser = argparse.ArgumentParser(description=description) + parser = argparse.ArgumentParser(description=description, color=True) parser.add_argument('-v', '--verbose', action='store_true', default=False, help='Verbose output') parser.add_argument('--filter', metavar='', @@ -2892,6 +3118,9 @@ def main(): '.tbz': 'bz2', '.tbz2': 'bz2', '.tb2': 'bz2', + # zstd + '.zst': 'zst', + '.tzst': 'zst', } tar_mode = 'w:' + compressions[ext] if ext in compressions else 'w' tar_files = args.create diff --git a/Lib/tempfile.py b/Lib/tempfile.py index 8036e93cd6d..5e3ccab5f48 100644 --- a/Lib/tempfile.py +++ b/Lib/tempfile.py @@ -438,11 +438,19 @@ class _TemporaryFileCloser: cleanup_called = False close_called = False - def __init__(self, file, name, delete=True, delete_on_close=True): + def __init__( + self, + file, + name, + delete=True, + delete_on_close=True, + warn_message="Implicitly cleaning up unknown file", + ): self.file = file self.name = name self.delete = delete self.delete_on_close = delete_on_close + self.warn_message = warn_message def cleanup(self, windows=(_os.name == 'nt'), unlink=_os.unlink): if not self.cleanup_called: @@ -470,7 +478,10 @@ def close(self): self.cleanup() def __del__(self): + close_called = self.close_called self.cleanup() + if not close_called: + _warnings.warn(self.warn_message, ResourceWarning) class _TemporaryFileWrapper: @@ -484,8 +495,17 @@ class _TemporaryFileWrapper: def __init__(self, file, name, delete=True, delete_on_close=True): self.file = file self.name = name - self._closer = _TemporaryFileCloser(file, name, delete, - delete_on_close) + self._closer = _TemporaryFileCloser( + file, + name, + delete, + delete_on_close, + warn_message=f"Implicitly cleaning up {self!r}", + ) + + def __repr__(self): + file = self.__dict__['file'] + return f"<{type(self).__name__} {file=}>" def __getattr__(self, name): # Attribute lookups are delegated to the underlying file diff --git a/Lib/test/_code_definitions.py b/Lib/test/_code_definitions.py new file mode 100644 index 00000000000..70c44da2ec6 --- /dev/null +++ b/Lib/test/_code_definitions.py @@ -0,0 +1,323 @@ + +def simple_script(): + assert True + + +def complex_script(): + obj = 'a string' + pickle = __import__('pickle') + def spam_minimal(): + pass + spam_minimal() + data = pickle.dumps(obj) + res = pickle.loads(data) + assert res == obj, (res, obj) + + +def script_with_globals(): + obj1, obj2 = spam(42) + assert obj1 == 42 + assert obj2 is None + + +def script_with_explicit_empty_return(): + return None + + +def script_with_return(): + return True + + +def spam_minimal(): + # no arg defaults or kwarg defaults + # no annotations + # no local vars + # no free vars + # no globals + # no builtins + # no attr access (names) + # no code + return + + +def spam_with_builtins(): + x = 42 + values = (42,) + checks = tuple(callable(v) for v in values) + res = callable(values), tuple(values), list(values), checks + print(res) + + +def spam_with_globals_and_builtins(): + func1 = spam + func2 = spam_minimal + funcs = (func1, func2) + checks = tuple(callable(f) for f in funcs) + res = callable(funcs), tuple(funcs), list(funcs), checks + print(res) + + +def spam_with_global_and_attr_same_name(): + try: + spam_minimal.spam_minimal + except AttributeError: + pass + + +def spam_full_args(a, b, /, c, d, *args, e, f, **kwargs): + return (a, b, c, d, e, f, args, kwargs) + + +def spam_full_args_with_defaults(a=-1, b=-2, /, c=-3, d=-4, *args, + e=-5, f=-6, **kwargs): + return (a, b, c, d, e, f, args, kwargs) + + +def spam_args_attrs_and_builtins(a, b, /, c, d, *args, e, f, **kwargs): + if args.__len__() > 2: + return None + return a, b, c, d, e, f, args, kwargs + + +def spam_returns_arg(x): + return x + + +def spam_raises(): + raise Exception('spam!') + + +def spam_with_inner_not_closure(): + def eggs(): + pass + eggs() + + +def spam_with_inner_closure(): + x = 42 + def eggs(): + print(x) + eggs() + + +def spam_annotated(a: int, b: str, c: object) -> tuple: + return a, b, c + + +def spam_full(a, b, /, c, d:int=1, *args, e, f:object=None, **kwargs) -> tuple: + # arg defaults, kwarg defaults + # annotations + # all kinds of local vars, except cells + # no free vars + # some globals + # some builtins + # some attr access (names) + x = args + y = kwargs + z = (a, b, c, d) + kwargs['e'] = e + kwargs['f'] = f + extras = list((x, y, z, spam, spam.__name__)) + return tuple(a, b, c, d, e, f, args, kwargs), extras + + +def spam(x): + return x, None + + +def spam_N(x): + def eggs_nested(y): + return None, y + return eggs_nested, x + + +def spam_C(x): + a = 1 + def eggs_closure(y): + return None, y, a, x + return eggs_closure, a, x + + +def spam_NN(x): + def eggs_nested_N(y): + def ham_nested(z): + return None, z + return ham_nested, y + return eggs_nested_N, x + + +def spam_NC(x): + a = 1 + def eggs_nested_C(y): + def ham_closure(z): + return None, z, y, a, x + return ham_closure, y + return eggs_nested_C, a, x + + +def spam_CN(x): + a = 1 + def eggs_closure_N(y): + def ham_C_nested(z): + return None, z + return ham_C_nested, y, a, x + return eggs_closure_N, a, x + + +def spam_CC(x): + a = 1 + def eggs_closure_C(y): + b = 2 + def ham_C_closure(z): + return None, z, b, y, a, x + return ham_C_closure, b, y, a, x + return eggs_closure_C, a, x + + +eggs_nested, *_ = spam_N(1) +eggs_closure, *_ = spam_C(1) +eggs_nested_N, *_ = spam_NN(1) +eggs_nested_C, *_ = spam_NC(1) +eggs_closure_N, *_ = spam_CN(1) +eggs_closure_C, *_ = spam_CC(1) + +ham_nested, *_ = eggs_nested_N(2) +ham_closure, *_ = eggs_nested_C(2) +ham_C_nested, *_ = eggs_closure_N(2) +ham_C_closure, *_ = eggs_closure_C(2) + + +TOP_FUNCTIONS = [ + # shallow + simple_script, + complex_script, + script_with_globals, + script_with_explicit_empty_return, + script_with_return, + spam_minimal, + spam_with_builtins, + spam_with_globals_and_builtins, + spam_with_global_and_attr_same_name, + spam_full_args, + spam_full_args_with_defaults, + spam_args_attrs_and_builtins, + spam_returns_arg, + spam_raises, + spam_with_inner_not_closure, + spam_with_inner_closure, + spam_annotated, + spam_full, + spam, + # outer func + spam_N, + spam_C, + spam_NN, + spam_NC, + spam_CN, + spam_CC, +] +NESTED_FUNCTIONS = [ + # inner func + eggs_nested, + eggs_closure, + eggs_nested_N, + eggs_nested_C, + eggs_closure_N, + eggs_closure_C, + # inner inner func + ham_nested, + ham_closure, + ham_C_nested, + ham_C_closure, +] +FUNCTIONS = [ + *TOP_FUNCTIONS, + *NESTED_FUNCTIONS, +] + +STATELESS_FUNCTIONS = [ + simple_script, + complex_script, + script_with_explicit_empty_return, + script_with_return, + spam, + spam_minimal, + spam_with_builtins, + spam_full_args, + spam_args_attrs_and_builtins, + spam_returns_arg, + spam_raises, + spam_annotated, + spam_with_inner_not_closure, + spam_with_inner_closure, + spam_N, + spam_C, + spam_NN, + spam_NC, + spam_CN, + spam_CC, + eggs_nested, + eggs_nested_N, + ham_nested, + ham_C_nested +] +STATELESS_CODE = [ + *STATELESS_FUNCTIONS, + script_with_globals, + spam_full_args_with_defaults, + spam_with_globals_and_builtins, + spam_with_global_and_attr_same_name, + spam_full, +] + +PURE_SCRIPT_FUNCTIONS = [ + simple_script, + complex_script, + script_with_explicit_empty_return, + spam_minimal, + spam_with_builtins, + spam_raises, + spam_with_inner_not_closure, + spam_with_inner_closure, +] +SCRIPT_FUNCTIONS = [ + *PURE_SCRIPT_FUNCTIONS, + script_with_globals, + spam_with_globals_and_builtins, + spam_with_global_and_attr_same_name, +] + + +# generators + +def gen_spam_1(*args): + for arg in args: + yield arg + + +def gen_spam_2(*args): + yield from args + + +async def async_spam(): + pass +coro_spam = async_spam() +coro_spam.close() + + +async def asyncgen_spam(*args): + for arg in args: + yield arg +asynccoro_spam = asyncgen_spam(1, 2, 3) + + +FUNCTION_LIKE = [ + gen_spam_1, + gen_spam_2, + async_spam, + asyncgen_spam, +] +FUNCTION_LIKE_APPLIED = [ + coro_spam, # actually FunctionType? + asynccoro_spam, # actually FunctionType? +] diff --git a/Lib/test/_test_atexit.py b/Lib/test/_test_atexit.py index 55d28083349..2e961d6a485 100644 --- a/Lib/test/_test_atexit.py +++ b/Lib/test/_test_atexit.py @@ -19,7 +19,9 @@ def assert_raises_unraisable(self, exc_type, func, *args): atexit.register(func, *args) atexit._run_exitfuncs() - self.assertEqual(cm.unraisable.object, func) + self.assertIsNone(cm.unraisable.object) + self.assertEqual(cm.unraisable.err_msg, + f'Exception ignored in atexit callback {func!r}') self.assertEqual(cm.unraisable.exc_type, exc_type) self.assertEqual(type(cm.unraisable.exc_value), exc_type) @@ -125,12 +127,61 @@ def func(): try: with support.catch_unraisable_exception() as cm: atexit._run_exitfuncs() - self.assertEqual(cm.unraisable.object, func) + self.assertIsNone(cm.unraisable.object) + self.assertEqual(cm.unraisable.err_msg, + f'Exception ignored in atexit callback {func!r}') self.assertEqual(cm.unraisable.exc_type, ZeroDivisionError) self.assertEqual(type(cm.unraisable.exc_value), ZeroDivisionError) finally: atexit.unregister(func) + def test_eq_unregister_clear(self): + # Issue #112127: callback's __eq__ may call unregister or _clear + class Evil: + def __eq__(self, other): + action(other) + return NotImplemented + + for action in atexit.unregister, lambda o: atexit._clear(): + with self.subTest(action=action): + atexit.register(lambda: None) + atexit.unregister(Evil()) + atexit._clear() + + def test_eq_unregister(self): + # Issue #112127: callback's __eq__ may call unregister + def f1(): + log.append(1) + def f2(): + log.append(2) + def f3(): + log.append(3) + + class Pred: + def __eq__(self, other): + nonlocal cnt + cnt += 1 + if cnt == when: + atexit.unregister(what) + if other is f2: + return True + return False + + for what, expected in ( + (f1, [3]), + (f2, [3, 1]), + (f3, [1]), + ): + for when in range(1, 4): + with self.subTest(what=what.__name__, when=when): + cnt = 0 + log = [] + for f in (f1, f2, f3): + atexit.register(f) + atexit.unregister(Pred()) + atexit._run_exitfuncs() + self.assertEqual(log, expected) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/_test_eintr.py b/Lib/test/_test_eintr.py index c8f04e9625c..b462f67b9c9 100644 --- a/Lib/test/_test_eintr.py +++ b/Lib/test/_test_eintr.py @@ -91,7 +91,7 @@ class OSEINTRTest(EINTRBaseTest): """ EINTR tests for the os module. """ def new_sleep_process(self): - code = 'import time; time.sleep(%r)' % self.sleep_time + code = f'import time; time.sleep({self.sleep_time!r})' return self.subprocess(code) def _test_wait_multiple(self, wait_func): @@ -123,36 +123,46 @@ def test_waitpid(self): def test_wait4(self): self._test_wait_single(lambda pid: os.wait4(pid, 0)) - def test_read(self): + def _interrupted_reads(self): + """Make a fd which will force block on read of expected bytes.""" rd, wr = os.pipe() self.addCleanup(os.close, rd) # wr closed explicitly by parent # the payload below are smaller than PIPE_BUF, hence the writes will be # atomic - datas = [b"hello", b"world", b"spam"] + data = [b"hello", b"world", b"spam"] code = '\n'.join(( 'import os, sys, time', '', 'wr = int(sys.argv[1])', - 'datas = %r' % datas, - 'sleep_time = %r' % self.sleep_time, + f'data = {data!r}', + f'sleep_time = {self.sleep_time!r}', '', - 'for data in datas:', + 'for item in data:', ' # let the parent block on read()', ' time.sleep(sleep_time)', - ' os.write(wr, data)', + ' os.write(wr, item)', )) proc = self.subprocess(code, str(wr), pass_fds=[wr]) with kill_on_error(proc): os.close(wr) - for data in datas: - self.assertEqual(data, os.read(rd, len(data))) + for datum in data: + yield rd, datum self.assertEqual(proc.wait(), 0) - @unittest.expectedFailure # TODO: RUSTPYTHON InterruptedError: [Errno 4] Interrupted system call + def test_read(self): + for fd, expected in self._interrupted_reads(): + self.assertEqual(expected, os.read(fd, len(expected))) + + def test_readinto(self): + for fd, expected in self._interrupted_reads(): + buffer = bytearray(len(expected)) + self.assertEqual(os.readinto(fd, buffer), len(expected)) + self.assertEqual(buffer, expected) + def test_write(self): rd, wr = os.pipe() self.addCleanup(os.close, wr) @@ -165,8 +175,8 @@ def test_write(self): 'import io, os, sys, time', '', 'rd = int(sys.argv[1])', - 'sleep_time = %r' % self.sleep_time, - 'data = b"x" * %s' % support.PIPE_MAX_SIZE, + f'sleep_time = {self.sleep_time!r}', + f'data = b"x" * {support.PIPE_MAX_SIZE}', 'data_len = len(data)', '', '# let the parent block on write()', @@ -179,8 +189,8 @@ def test_write(self): '', 'value = read_data.getvalue()', 'if value != data:', - ' raise Exception("read error: %s vs %s bytes"', - ' % (len(value), data_len))', + ' raise Exception(f"read error: {len(value)}' + ' vs {data_len} bytes")', )) proc = self.subprocess(code, str(rd), pass_fds=[rd]) @@ -203,33 +213,33 @@ def _test_recv(self, recv_func): # wr closed explicitly by parent # single-byte payload guard us against partial recv - datas = [b"x", b"y", b"z"] + data = [b"x", b"y", b"z"] code = '\n'.join(( 'import os, socket, sys, time', '', 'fd = int(sys.argv[1])', - 'family = %s' % int(wr.family), - 'sock_type = %s' % int(wr.type), - 'datas = %r' % datas, - 'sleep_time = %r' % self.sleep_time, + f'family = {int(wr.family)}', + f'sock_type = {int(wr.type)}', + f'data = {data!r}', + f'sleep_time = {self.sleep_time!r}', '', 'wr = socket.fromfd(fd, family, sock_type)', 'os.close(fd)', '', 'with wr:', - ' for data in datas:', + ' for item in data:', ' # let the parent block on recv()', ' time.sleep(sleep_time)', - ' wr.sendall(data)', + ' wr.sendall(item)', )) fd = wr.fileno() proc = self.subprocess(code, str(fd), pass_fds=[fd]) with kill_on_error(proc): wr.close() - for data in datas: - self.assertEqual(data, recv_func(rd, len(data))) + for item in data: + self.assertEqual(item, recv_func(rd, len(item))) self.assertEqual(proc.wait(), 0) def test_recv(self): @@ -251,10 +261,10 @@ def _test_send(self, send_func): 'import os, socket, sys, time', '', 'fd = int(sys.argv[1])', - 'family = %s' % int(rd.family), - 'sock_type = %s' % int(rd.type), - 'sleep_time = %r' % self.sleep_time, - 'data = b"xyz" * %s' % (support.SOCK_MAX_SIZE // 3), + f'family = {int(rd.family)}', + f'sock_type = {int(rd.type)}', + f'sleep_time = {self.sleep_time!r}', + f'data = b"xyz" * {support.SOCK_MAX_SIZE // 3}', 'data_len = len(data)', '', 'rd = socket.fromfd(fd, family, sock_type)', @@ -270,8 +280,8 @@ def _test_send(self, send_func): ' n += rd.recv_into(memoryview(received_data)[n:])', '', 'if received_data != data:', - ' raise Exception("recv error: %s vs %s bytes"', - ' % (len(received_data), data_len))', + ' raise Exception(f"recv error: {len(received_data)}' + ' vs {data_len} bytes")', )) fd = rd.fileno() @@ -303,9 +313,9 @@ def test_accept(self): code = '\n'.join(( 'import socket, time', '', - 'host = %r' % socket_helper.HOST, - 'port = %s' % port, - 'sleep_time = %r' % self.sleep_time, + f'host = {socket_helper.HOST!r}', + f'port = {port}', + f'sleep_time = {self.sleep_time!r}', '', '# let parent block on accept()', 'time.sleep(sleep_time)', @@ -333,15 +343,15 @@ def _test_open(self, do_open_close_reader, do_open_close_writer): os_helper.unlink(filename) try: os.mkfifo(filename) - except PermissionError as e: - self.skipTest('os.mkfifo(): %s' % e) + except PermissionError as exc: + self.skipTest(f'os.mkfifo(): {exc!r}') self.addCleanup(os_helper.unlink, filename) code = '\n'.join(( 'import os, time', '', - 'path = %a' % filename, - 'sleep_time = %r' % self.sleep_time, + f'path = {filename!a}', + f'sleep_time = {self.sleep_time!r}', '', '# let the parent block', 'time.sleep(sleep_time)', @@ -381,7 +391,6 @@ def test_os_open(self): class TimeEINTRTest(EINTRBaseTest): """ EINTR tests for the time module. """ - @unittest.expectedFailure # TODO: RUSTPYTHON def test_sleep(self): t0 = time.monotonic() time.sleep(self.sleep_time) @@ -400,21 +409,20 @@ class SignalEINTRTest(EINTRBaseTest): def check_sigwait(self, wait_func): signum = signal.SIGUSR1 - pid = os.getpid() old_handler = signal.signal(signum, lambda *args: None) self.addCleanup(signal.signal, signum, old_handler) code = '\n'.join(( 'import os, time', - 'pid = %s' % os.getpid(), - 'signum = %s' % int(signum), - 'sleep_time = %r' % self.sleep_time, + f'pid = {os.getpid()}', + f'signum = {int(signum)}', + f'sleep_time = {self.sleep_time!r}', 'time.sleep(sleep_time)', 'os.kill(pid, signum)', )) - old_mask = signal.pthread_sigmask(signal.SIG_BLOCK, [signum]) + signal.pthread_sigmask(signal.SIG_BLOCK, [signum]) self.addCleanup(signal.pthread_sigmask, signal.SIG_UNBLOCK, [signum]) proc = self.subprocess(code) @@ -452,7 +460,7 @@ def test_select(self): self.stop_alarm() self.check_elapsed_time(dt) - @unittest.skip('TODO: RUSTPYTHON timed out at the 10 minute mark') + @unittest.skip("TODO: RUSTPYTHON; timed out at the 10 minute mark") @unittest.skipIf(sys.platform == "darwin", "poll may fail on macOS; see issue #28087") @unittest.skipUnless(hasattr(select, 'poll'), 'need select.poll') @@ -534,14 +542,14 @@ def _lock(self, lock_func, lock_name): self.check_elapsed_time(dt) proc.wait() - @unittest.expectedFailure # TODO: RUSTPYTHON InterruptedError: [Errno 4] Interrupted system call # Issue 35633: See https://bugs.python.org/issue35633#msg333662 # skip test rather than accept PermissionError from all platforms + @unittest.expectedFailure # TODO: RUSTPYTHON; InterruptedError: [Errno 4] Interrupted system call @unittest.skipIf(platform.system() == "AIX", "AIX returns PermissionError") def test_lockf(self): self._lock(fcntl.lockf, "lockf") - @unittest.expectedFailure # TODO: RUSTPYTHON InterruptedError: [Errno 4] Interrupted system call + @unittest.expectedFailure # TODO: RUSTPYTHON; InterruptedError: [Errno 4] Interrupted system call def test_flock(self): self._lock(fcntl.flock, "flock") diff --git a/Lib/test/_test_gc_fast_cycles.py b/Lib/test/_test_gc_fast_cycles.py new file mode 100644 index 00000000000..4e2c7d72a02 --- /dev/null +++ b/Lib/test/_test_gc_fast_cycles.py @@ -0,0 +1,48 @@ +# Run by test_gc. +from test import support +import _testinternalcapi +import gc +import unittest + +class IncrementalGCTests(unittest.TestCase): + + # Use small increments to emulate longer running process in a shorter time + @support.gc_threshold(200, 10) + def test_incremental_gc_handles_fast_cycle_creation(self): + + class LinkedList: + + #Use slots to reduce number of implicit objects + __slots__ = "next", "prev", "surprise" + + def __init__(self, next=None, prev=None): + self.next = next + if next is not None: + next.prev = self + self.prev = prev + if prev is not None: + prev.next = self + + def make_ll(depth): + head = LinkedList() + for i in range(depth): + head = LinkedList(head, head.prev) + return head + + head = make_ll(1000) + + assert(gc.isenabled()) + olds = [] + initial_heap_size = _testinternalcapi.get_tracked_heap_size() + for i in range(20_000): + newhead = make_ll(20) + newhead.surprise = head + olds.append(newhead) + if len(olds) == 20: + new_objects = _testinternalcapi.get_tracked_heap_size() - initial_heap_size + self.assertLess(new_objects, 27_000, f"Heap growing. Reached limit after {i} iterations") + del olds[:] + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 0855e384f24..a009cb8cadd 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -1459,7 +1459,7 @@ def _acquire_release(lock, timeout, l=None, n=1): for _ in range(n): lock.release() - @unittest.skip("TODO: RUSTPYTHON; flaky timeout") + @unittest.skip("TODO: RUSTPYTHON; flaky timeout - thread start latency") def test_repr_rlock(self): if self.TYPE != 'processes': self.skipTest('test not appropriate for {}'.format(self.TYPE)) @@ -4057,8 +4057,6 @@ def test_heap(self): self.assertEqual(len(heap._allocated_blocks), 0, heap._allocated_blocks) self.assertEqual(len(heap._len_to_seq), 0) - # TODO: RUSTPYTHON - gc.enable() not implemented - @unittest.expectedFailure def test_free_from_gc(self): # Check that freeing of blocks by the garbage collector doesn't deadlock # (issue #12352). @@ -4417,7 +4415,6 @@ def test_shared_memory_across_processes(self): sms.close() - @unittest.skip("TODO: RUSTPYTHON; flaky") @unittest.skipIf(os.name != "posix", "not feasible in non-posix platforms") def test_shared_memory_SharedMemoryServer_ignores_sigint(self): # bpo-36368: protect SharedMemoryManager server process from @@ -4816,8 +4813,8 @@ def test_finalize(self): result = [obj for obj in iter(conn.recv, 'STOP')] self.assertEqual(result, ['a', 'b', 'd10', 'd03', 'd02', 'd01', 'e']) - # TODO: RUSTPYTHON - gc.get_threshold() and gc.set_threshold() not implemented - @unittest.expectedFailure + # TODO: RUSTPYTHON; SIGSEGV due to dict thread-safety issue under aggressive GC + @unittest.skip("TODO: RUSTPYTHON") @support.requires_resource('cpu') def test_thread_safety(self): # bpo-24484: _run_finalizers() should be thread-safe @@ -5445,8 +5442,6 @@ def run_in_child(cls, start_method): flags = (tuple(sys.flags), grandchild_flags) print(json.dumps(flags)) - # TODO: RUSTPYTHON - SyntaxError in subprocess after fork - @unittest.expectedFailure def test_flags(self): import json # start child process using unusual flags @@ -6592,7 +6587,8 @@ def tearDownClass(cls): # cycles. Trigger a garbage collection to break these cycles. test.support.gc_collect() - processes = set(multiprocessing.process._dangling) - set(cls.dangling[0]) + # TODO: RUSTPYTHON: Filter out stopped processes since gc.collect() is a no-op + processes = {p for p in multiprocessing.process._dangling if p.is_alive()} - {p for p in cls.dangling[0] if p.is_alive()} if processes: test.support.environment_altered = True support.print_warning(f'Dangling processes: {processes}') @@ -6789,7 +6785,8 @@ def tearDownModule(): multiprocessing.set_start_method(old_start_method[0], force=True) # pause a bit so we don't get warning about dangling threads/processes - processes = set(multiprocessing.process._dangling) - set(dangling[0]) + # TODO: RUSTPYTHON: Filter out stopped processes since gc.collect() is a no-op + processes = {p for p in multiprocessing.process._dangling if p.is_alive()} - {p for p in dangling[0] if p.is_alive()} if processes: need_sleep = True test.support.environment_altered = True diff --git a/Lib/test/audiodata/pluck-alaw.aifc b/Lib/test/audiodata/pluck-alaw.aifc deleted file mode 100644 index 3b7fbd2af75..00000000000 Binary files a/Lib/test/audiodata/pluck-alaw.aifc and /dev/null differ diff --git a/Lib/test/audiodata/pluck-pcm16.aiff b/Lib/test/audiodata/pluck-pcm16.aiff deleted file mode 100644 index 6c8c40d1409..00000000000 Binary files a/Lib/test/audiodata/pluck-pcm16.aiff and /dev/null differ diff --git a/Lib/test/audiodata/pluck-pcm16.au b/Lib/test/audiodata/pluck-pcm16.au deleted file mode 100644 index 398f07f0719..00000000000 Binary files a/Lib/test/audiodata/pluck-pcm16.au and /dev/null differ diff --git a/Lib/test/audiodata/pluck-pcm24.aiff b/Lib/test/audiodata/pluck-pcm24.aiff deleted file mode 100644 index 8eba145a44d..00000000000 Binary files a/Lib/test/audiodata/pluck-pcm24.aiff and /dev/null differ diff --git a/Lib/test/audiodata/pluck-pcm24.au b/Lib/test/audiodata/pluck-pcm24.au deleted file mode 100644 index 0bb230418a3..00000000000 Binary files a/Lib/test/audiodata/pluck-pcm24.au and /dev/null differ diff --git a/Lib/test/audiodata/pluck-pcm32.aiff b/Lib/test/audiodata/pluck-pcm32.aiff deleted file mode 100644 index 46ac0373f6a..00000000000 Binary files a/Lib/test/audiodata/pluck-pcm32.aiff and /dev/null differ diff --git a/Lib/test/audiodata/pluck-pcm32.au b/Lib/test/audiodata/pluck-pcm32.au deleted file mode 100644 index 92ee5965e40..00000000000 Binary files a/Lib/test/audiodata/pluck-pcm32.au and /dev/null differ diff --git a/Lib/test/audiodata/pluck-pcm8.aiff b/Lib/test/audiodata/pluck-pcm8.aiff deleted file mode 100644 index 5de4f3b2d87..00000000000 Binary files a/Lib/test/audiodata/pluck-pcm8.aiff and /dev/null differ diff --git a/Lib/test/audiodata/pluck-pcm8.au b/Lib/test/audiodata/pluck-pcm8.au deleted file mode 100644 index b7172c8f234..00000000000 Binary files a/Lib/test/audiodata/pluck-pcm8.au and /dev/null differ diff --git a/Lib/test/audiodata/pluck-ulaw.aifc b/Lib/test/audiodata/pluck-ulaw.aifc deleted file mode 100644 index 3085cf097fb..00000000000 Binary files a/Lib/test/audiodata/pluck-ulaw.aifc and /dev/null differ diff --git a/Lib/test/audiodata/pluck-ulaw.au b/Lib/test/audiodata/pluck-ulaw.au deleted file mode 100644 index 11103535c6b..00000000000 Binary files a/Lib/test/audiodata/pluck-ulaw.au and /dev/null differ diff --git a/Lib/test/certdata/allsans.pem b/Lib/test/certdata/allsans.pem index 02f2c2e6346..f6a63b9219d 100644 --- a/Lib/test/certdata/allsans.pem +++ b/Lib/test/certdata/allsans.pem @@ -1,42 +1,42 @@ -----BEGIN PRIVATE KEY----- -MIIG/QIBADANBgkqhkiG9w0BAQEFAASCBucwggbjAgEAAoIBgQCczEVv5D2UDtn6 -DMmZ/uCWCLyL+K5xTZp/5j3cyISoaTuU1Ku3kD97eLgpHj4Fgk5ZJi21zsQqepCj -jAhBk6tj6RYUcnMbb8MuxUkQMEDW+5LfSyp+HCaetlHosWdhEDqX4kpJ5ajBwNRt -07mxQExtC4kcno0ut9rG5XzLN29XpCpRHlFFrntOgQAEoiz9/fc8qaTgb37RgGYP -Qsxh7PcRDRe4ZGx1l06Irr8Y+2W50zWCfkwCS3DaLDOKIjSOfPHNqmfcfsTpzrj8 -330cdPklrMIuiBv+iGklCjkPZJiEhxvY2k6ERM4HAxxuPCivrH5MCeMNYvBVUcLr -GROm7JRXRllI/XubwwoAaAb+y+dZtCZ9AnzHIb+nyKiJxWAjzjR+QPL6jHrVWBVA -WTc83YP5FvxUXMfY3sVv9tNSCV3cpYOW5+iXcQzLuczXnOLRYk7p9wkb0/hk9KuK -4BMA90eBhvFMCFgHJ1/xJg2nFmBHPo/xbcwPG/ma5T/McA8mAlECAwEAAQKCAYAB -m29nxPNjod5Wm4xydWQYbZj/J0qkcyru/i1qpqyDbGa1sRNcg5A/A/8BPuPcWxhR -/hvwVeD5XX2/i2cnQuv6D3DQP1cSNCxQPanwzknP2k7IVqUmG0RDErPWuoDIhCnR -ljp0NPQsnj0fLhEkcbgG0xwx7KceUDigGsiTbatIvvBHGhQzrmTpqlVVdtMWvGRt -HQEJYuMuIw6IwALHyy3CITv5wh/Bec5OhNoFF8iUZceR4ZkGWf8bYWIa25xlzH6K -4rhOOh1G2ObHHTjhZq4mGXTHY1MEkAxXKWlR3DJc0Lh5E1UETSI6WBHWRb08iwQ5 -AkLOPyMpt08xHFWbJqywvlxenpri+gjY3xbXqGNhyDYWHZqlQmJVnzxoUOuuHi2R -dO86IckUc4Thjbm7a6AilL9t8juNZvyeQUVgtVi25uPkm/cK6r5vo8y4UOUU41EN -NOathlF69gh93il4t6zJW9jPV2WENv1H/vhKUWKW6cabX3vy4rANwy3q4V//GDEC -gcEAuniGCHaEdSjV2sUHyt/yrCLbU6+eLTfNk6AQyXJk6Wrvj6g3gx90ewEq5i/C -ukmSKDslr9pupq8Z/UNfYHZfJfpwEsYvIZ8DdFSd62/h66DhIoEn1v3Lwt+aexgX -yGJHF0BG9JA2CU5Z5NGjlnQYqQBobO9uZMq62l15Ig1MAMHGL0ZYVvOqGZD7XvtC -4UnclK5kjp51Vd5rydEQxyi5qkyLl9Q6T0FQXOphGIOd8ifYeUGe7YC72cFPevdx -wDXZAoHBANdDVvCMrjmzdrS6td39/2nHTeerFPbsOe2LIQYzqjeEe6GWqd2NL9NZ -bk3/cAuVgbWtdvSQQhhmSqOC7JZic4hbZb3lK6v/sr4F/Zu0CfAu80swWFMeS7vq -eQeYzN4w4dKpJArvU3ll7N9AlZhdlYkbPf0WdeOIjZawdAOxNtNe0O+j+5MsXR59 -qkULatumhcKUnqxFCiVHzy21CVJtRzrtu6oGoSdFbmG82eSJ1rPXiuuDnCyzjyMW -iClYRM4NOQKBwERnO/vUxihYT4LOLlqcpl/A9aYQUT0TMGWMHTxYq2343WJceeiu -3ELXHc6NDKjbnjMF54BH57lbmHQQh+dR5PuAkCZC7z0tIM5G0Bty0nRmcs/+gwfZ -2Cpnbjrjjq3iZ2O/H4hNcpUdWdqXkKP7eKReUvBLMLrmp369NVdpe0z3yGTFMFjN -T8PLLHsePt14A+PCyX6L4E0cp3vEJpx4cwtmwvpyTuWN9xXuoKmmdoVDWqS4jr1f -MQnjYO2h4ed5mQKBwGVttWli4DUP+r7tuwP+ynptDqg6VIaEiEcFZ2okre+63QYm -l6NtAzvyx6a41XKf355bPdG+p2YXzNN+vTue6BE3/5iagxloQjCHYhgbnRMvDDRB -c1y2ybihoqWRufZ30fARAoqkehCZliMbq2E/t1YDIBJAowuzLAP04LVcqxitdIV2 -HvQZ00aqr7AY0SDuNdiZbqp9XWpzi4td4iaUlxuNKP/UX9rBPGGROpoU2LWkujB+ -svfdI3TFCSNyE/mDAQKBwQCP++WZKxExrSFRk3W+TcHKHZb2pusfoPWE7WH6EnDW -dkTZpa3PZaf0xgeglmNBv4Paxw2eMPsIhyNv62XY/6GbY6VJWRyx/s+NsazeP4ji -xUOufnwTePjYw6x0pcl6BknZrHn8LCJU741h0yTum8cDdNfRKdc0AMy0gVXk4ZTG -2cAtbEcWb3J+a5kYf6mp5yx3BNwtewkGZhc2VuQ9mQNbMmOOS/pHQQTRWcxsQwyt -GPAhMKawjrL1KFmu7vIqDSw= +MIIG/QIBADANBgkqhkiG9w0BAQEFAASCBucwggbjAgEAAoIBgQCYEY+lIwS/w+gA +U5PfxSeL4qMF5TtFkrifL0oQaCjWFb8jeX2Rb1tKaObyrYmTURVWbMkI1LJOQWYt +GlANo2mvf4VD1wZZ/x/Tmylxe3A7itSTZgZuGXLHtFP9wk7yS46BlVcNq3ZoOi7k +k8NxTiUqgsmLg3Nj+Ifk8OI7c1BcP2pa1wZO/l3PTWqtFSLsFEvOK5N6ZPqILAUi +5vEkOiFefxVHHvD4N3IWSFgSwiDi9KhntdRwa+axsZxjq/o70FjQO5q5Quym9aDq +qo47PMykEb1gdTBQK1mPliMKsLi5zycI/ypm1PEcH53+9/F+U2B1bmvA/9PBx8T7 +hUyclBbQHCmdcXIYCgFr5AbGS3jFeVPiF7hCPj1k/jcpayXZAtWmqJTlkXKGbM68 +AgM/DHO0fGCYGRUw2vhV3ZIRCm0BqwforeFh+2oMyZjDjvo/uZjZNazdpcwvOy0c +lFkORwVI0zTIixXYSV4xN1BX821JCuhYubro89m+MSjxO14qgs0CAwEAAQKCAYAZ +txqF7+qNLSlN6NRASnw6BQzRYebIiJus25fYN2z0awAEFTbdQan75sprLlpt7Y/A +qivC0QkH/7oyFVmFPOWR1mwoQTPjQyfUJlB3Tsr3Xq488MtUkfwddkqfxlyIT6ud +ES6D8sNWs8QbRjuOLQtO6YgAji2UscH1PqDbMdBckSLAks0Pzab6d9p6w3DA4FvD +VQ4e6/WL0nnZ4ZjUqfnbm3zzJnHUX7fsubYfEfHyvzG9O/vdOPntgZ3zIvFxbPVc +/Dp8MRNgfQoRgAsc7VBQQifFNvva9SjvRsXwE7zQLQh9JszFpHWqIMevGwjdH8dO +1hHESqkRHL0sYwsqoGqbaHFrJVEA2Ljuxs/UsH6G2U6NuDVaYkqM8UwH0+EqGFtF +9qP8oh6Yvp+szI5yET4brW0Rlk8r2UqvQ5iSSl8SSWsNKR2wRs0PM0aNm13TDJmt +NOuxd/0ranG+NvBMdtLCwsoEplYDaaczYv9gbKHT64OlFAdGBkOytbS7eqOojmcC +gcEA0uQb9bPKbiMYSEJMAalOlruzvMiHBRW420IEuGyHJtocn/RK804Jj6gzu+WD +GT81rB9NXAt/3s6WDMSdrGrrBh/CQZJs50ok9hej0f6gsq1JNsWJ/c10rU/mqrTn +s++pdaRqbd+bgM/N6h79tJi3j4SgB9rJqjC+7qWlDJFWZOoDR59elc10xDA0AE0a +K6UvsJJGk4/v715K1Uh2qU5HYupuijNc0m3z2OMZ6zxTNNUdbSL4XF5X9e2+BwMQ +YrRzAoHBALiYeYWsmHGRdg++/p/9SE86YwyEYr+zrUiWenIPC1W5WQlkdFQqALmg +kjcXAXFFouC/DnKG65YOfGq772ekFuw06Lm4DEotQP02pESfAP9ulVufK2yJtbxi +m0PeN3JRrs/H72VW9hUSxCafNRyBeZ6nsfzEfEkPOdc+lRGeMHT9C7r7Y1asNk8A +Zl7wKZgv+Yvn8xhHWQj5z/T6K2z5CXFn6FERc0qV8+DggE9m3CjaaNxft6IodfHD +h1iakNtbvwKBwQCpWr/NRy131r0IQh0xdFoFGAUVxF8RSUli4hhSVe0O2TcFiLOr +wW5SK/wnlv75hlY+vABuu1lbfsDmzfnk3RORnm1sJP9JmbQm4AMRfw5jjl7uGiJf +a9+X0kNlsNMlH4ARVhCV3WzOO5KbwXlxzvYRzaqJxDwQbQbXNLRfbFNZxMcPfD8D +w7NSXXdVCpXKmOO8QytkEsHWkv07W+7WtWMEX0iXuPmAjwW0lWNaEd6r3by8yMlz +u9udRedFUEOXUFsCgcAX0g00c75EQXoTtBjVenC/UJCBh//aLwx4Znqsh0Z2LHHR +5XWher4XNiJIG57jCBJpoB30J3b1KS9i8peFL0aJ+pXhiV+EnuxZAJkYBdCyJYn+ +hb6rxeV+xta0XlOXW/UL+QfqcttUgtRvC3JmGEsibw9nx88l+mIDZZ8E4/3qytCd +s1zxTU3AyhNrwuALNH2mUSssgeB6aQot2a6K5GQUj00KUQ8om8sZxL6qAGL+npiT +f4KJ2WDG7u1jQKbat68CgcBONTGp3M3pdwc3OV/PMFq1+h1/BuyW/5qUtbsvXkrD +DZyXGY2SxJFMipakzOppusaQOYmO+VXKVDRJm8UG6fa2cdXjwm2kDxHB5E+v59VI ++uM8ErX6L52I0+rdoOU6AwpXyiVW5AXzMInOGHF4B7zJ1SA25RwVG44LbDw1cehW +MALXUdVCaegmvCCFq5TQlrlaEktxWk6Th92mXWbNbwUHlIGpcjEkASFvt90aBXWZ +w0YCQFVxx/K95nzKyRTjHTQ= -----END PRIVATE KEY----- Certificate: Data: @@ -47,38 +47,38 @@ Certificate: Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server Validity Not Before: Aug 29 14:23:16 2018 GMT - Not After : Oct 28 14:23:16 2037 GMT + Not After : Oct 28 14:23:16 2525 GMT Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=allsans Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (3072 bit) Modulus: - 00:9c:cc:45:6f:e4:3d:94:0e:d9:fa:0c:c9:99:fe: - e0:96:08:bc:8b:f8:ae:71:4d:9a:7f:e6:3d:dc:c8: - 84:a8:69:3b:94:d4:ab:b7:90:3f:7b:78:b8:29:1e: - 3e:05:82:4e:59:26:2d:b5:ce:c4:2a:7a:90:a3:8c: - 08:41:93:ab:63:e9:16:14:72:73:1b:6f:c3:2e:c5: - 49:10:30:40:d6:fb:92:df:4b:2a:7e:1c:26:9e:b6: - 51:e8:b1:67:61:10:3a:97:e2:4a:49:e5:a8:c1:c0: - d4:6d:d3:b9:b1:40:4c:6d:0b:89:1c:9e:8d:2e:b7: - da:c6:e5:7c:cb:37:6f:57:a4:2a:51:1e:51:45:ae: - 7b:4e:81:00:04:a2:2c:fd:fd:f7:3c:a9:a4:e0:6f: - 7e:d1:80:66:0f:42:cc:61:ec:f7:11:0d:17:b8:64: - 6c:75:97:4e:88:ae:bf:18:fb:65:b9:d3:35:82:7e: - 4c:02:4b:70:da:2c:33:8a:22:34:8e:7c:f1:cd:aa: - 67:dc:7e:c4:e9:ce:b8:fc:df:7d:1c:74:f9:25:ac: - c2:2e:88:1b:fe:88:69:25:0a:39:0f:64:98:84:87: - 1b:d8:da:4e:84:44:ce:07:03:1c:6e:3c:28:af:ac: - 7e:4c:09:e3:0d:62:f0:55:51:c2:eb:19:13:a6:ec: - 94:57:46:59:48:fd:7b:9b:c3:0a:00:68:06:fe:cb: - e7:59:b4:26:7d:02:7c:c7:21:bf:a7:c8:a8:89:c5: - 60:23:ce:34:7e:40:f2:fa:8c:7a:d5:58:15:40:59: - 37:3c:dd:83:f9:16:fc:54:5c:c7:d8:de:c5:6f:f6: - d3:52:09:5d:dc:a5:83:96:e7:e8:97:71:0c:cb:b9: - cc:d7:9c:e2:d1:62:4e:e9:f7:09:1b:d3:f8:64:f4: - ab:8a:e0:13:00:f7:47:81:86:f1:4c:08:58:07:27: - 5f:f1:26:0d:a7:16:60:47:3e:8f:f1:6d:cc:0f:1b: - f9:9a:e5:3f:cc:70:0f:26:02:51 + 00:98:11:8f:a5:23:04:bf:c3:e8:00:53:93:df:c5: + 27:8b:e2:a3:05:e5:3b:45:92:b8:9f:2f:4a:10:68: + 28:d6:15:bf:23:79:7d:91:6f:5b:4a:68:e6:f2:ad: + 89:93:51:15:56:6c:c9:08:d4:b2:4e:41:66:2d:1a: + 50:0d:a3:69:af:7f:85:43:d7:06:59:ff:1f:d3:9b: + 29:71:7b:70:3b:8a:d4:93:66:06:6e:19:72:c7:b4: + 53:fd:c2:4e:f2:4b:8e:81:95:57:0d:ab:76:68:3a: + 2e:e4:93:c3:71:4e:25:2a:82:c9:8b:83:73:63:f8: + 87:e4:f0:e2:3b:73:50:5c:3f:6a:5a:d7:06:4e:fe: + 5d:cf:4d:6a:ad:15:22:ec:14:4b:ce:2b:93:7a:64: + fa:88:2c:05:22:e6:f1:24:3a:21:5e:7f:15:47:1e: + f0:f8:37:72:16:48:58:12:c2:20:e2:f4:a8:67:b5: + d4:70:6b:e6:b1:b1:9c:63:ab:fa:3b:d0:58:d0:3b: + 9a:b9:42:ec:a6:f5:a0:ea:aa:8e:3b:3c:cc:a4:11: + bd:60:75:30:50:2b:59:8f:96:23:0a:b0:b8:b9:cf: + 27:08:ff:2a:66:d4:f1:1c:1f:9d:fe:f7:f1:7e:53: + 60:75:6e:6b:c0:ff:d3:c1:c7:c4:fb:85:4c:9c:94: + 16:d0:1c:29:9d:71:72:18:0a:01:6b:e4:06:c6:4b: + 78:c5:79:53:e2:17:b8:42:3e:3d:64:fe:37:29:6b: + 25:d9:02:d5:a6:a8:94:e5:91:72:86:6c:ce:bc:02: + 03:3f:0c:73:b4:7c:60:98:19:15:30:da:f8:55:dd: + 92:11:0a:6d:01:ab:07:e8:ad:e1:61:fb:6a:0c:c9: + 98:c3:8e:fa:3f:b9:98:d9:35:ac:dd:a5:cc:2f:3b: + 2d:1c:94:59:0e:47:05:48:d3:34:c8:8b:15:d8:49: + 5e:31:37:50:57:f3:6d:49:0a:e8:58:b9:ba:e8:f3: + d9:be:31:28:f1:3b:5e:2a:82:cd Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Alternative Name: @@ -90,9 +90,9 @@ Certificate: X509v3 Basic Constraints: critical CA:FALSE X509v3 Subject Key Identifier: - 31:5E:C0:5E:2F:47:FF:8B:92:F9:EE:3D:B1:87:D0:53:75:3B:B1:48 + 73:0D:4E:1F:4A:EC:F2:53:0F:53:FC:85:6F:CF:82:CD:A3:E8:11:8D X509v3 Authority Key Identifier: - keyid:F3:EC:94:8E:F2:8E:30:C4:8E:68:C2:BF:8E:6A:19:C0:C1:9F:76:65 + keyid:C0:0A:2B:28:43:DE:5F:C9:7D:47:E5:47:9B:36:F2:65:8C:67:3B:E2 DirName:/C=XY/O=Python Software Foundation CA/CN=our-ca-server serial:CB:2D:80:99:5A:69:52:5B Authority Information Access: @@ -103,65 +103,65 @@ Certificate: URI:http://testca.pythontest.net/testca/revocation.crl Signature Algorithm: sha256WithRSAEncryption Signature Value: - 72:42:a6:fc:ee:3c:21:47:05:33:e8:8c:6b:27:07:4a:ed:e2: - 81:47:96:79:43:ff:0f:ef:5a:06:aa:4c:01:70:5b:21:c4:b7: - 5d:17:29:c8:10:02:c3:08:7b:8c:86:56:9e:e9:7c:6e:a8:b6: - 26:13:9e:1e:1f:93:66:85:67:63:9e:08:fb:55:39:56:82:f5: - be:0c:38:1e:eb:c4:54:b2:a7:7b:18:55:bb:00:87:43:50:50: - bb:e1:29:10:cf:3d:c9:07:c7:d2:5d:b6:45:68:1f:d6:de:00: - 96:3e:29:73:f6:22:70:21:a2:ba:68:28:94:ec:37:bc:a7:00: - 70:58:4e:d1:48:ae:ef:8d:11:a4:6e:10:2f:92:83:07:e2:76: - ac:bf:4f:bb:d6:9f:47:9e:a4:02:03:16:f8:a8:0a:3d:67:17: - 31:44:0e:68:d0:d3:24:d5:e7:bf:67:30:8f:88:97:92:0a:1e: - d7:74:df:7e:7b:4c:c6:d9:c3:84:92:2b:a0:89:11:08:4c:dd: - 32:49:df:36:23:d4:63:56:e4:f1:68:5a:6f:a0:c3:3c:e2:36: - ee:f3:46:60:78:4d:76:a5:5a:4a:61:c6:f8:ae:18:68:c2:8d: - 0e:2f:76:50:bb:be:b9:56:f1:04:5c:ac:ad:d7:d6:a4:1e:45: - 45:52:f4:10:a2:0f:9b:e3:d9:73:17:b6:52:42:a6:5b:c9:e9: - 8d:60:74:68:d0:1f:7a:ce:01:8e:9e:55:cb:cf:64:c1:cc:9a: - 72:aa:b4:5f:b5:55:13:41:10:51:a0:2c:a5:5b:43:12:ca:cc: - b7:c4:ac:f2:6f:72:fd:0d:50:6a:d6:81:c1:91:93:21:fe:de: - 9a:be:e5:3c:2a:98:95:a1:42:f8:f2:5c:75:c6:f1:fd:11:b1: - 22:26:33:5b:43:63:21:06:61:d2:cd:04:f3:30:c6:a8:3f:17: - d3:05:a3:87:45:2e:52:1e:51:88:e3:59:4c:78:51:b0:7b:b4: - 58:d9:27:22:6e:8c + a5:3d:12:87:b9:3b:3e:9c:1c:59:fb:d5:38:22:49:61:f3:c3: + 11:53:4b:4e:63:af:f2:3d:ef:24:67:45:bc:74:5c:4a:65:c5: + b4:bb:fe:84:b8:b6:ca:7d:fc:aa:ff:80:ae:67:1f:cb:c3:cd: + 8f:f9:75:8c:f9:d3:3f:db:f6:81:d8:06:42:c3:5d:a9:1e:a3: + 81:7d:57:ac:97:d9:bd:c8:ce:1e:ec:74:d7:94:d5:df:b1:ad: + ce:84:14:5c:8c:45:a4:a8:eb:67:ab:16:57:61:15:86:ae:11: + 1e:b5:10:42:de:84:76:88:9b:37:12:aa:a6:77:42:75:b4:c0: + 04:b3:75:45:e0:d7:aa:34:e3:07:c5:ed:f8:4e:f0:39:99:1f: + 5b:d8:4e:0c:ad:64:6d:09:07:3f:e3:e1:9f:1b:65:07:96:59: + 9a:b5:f1:4d:c3:ec:f7:32:a1:05:94:d1:0b:18:54:3c:67:cf: + 38:f5:2b:ec:cb:bd:79:be:f7:1b:b7:71:3f:c6:44:80:7f:00: + dc:3d:a0:07:c0:b5:1c:fb:52:f6:a0:f8:92:c6:c6:73:07:c5: + ca:0b:04:7c:55:51:e3:ba:93:32:17:bd:61:ae:cf:13:e4:5e: + 03:b2:51:11:c7:68:f5:08:b6:0e:57:49:11:3c:e3:f4:0e:e1: + 96:20:44:28:31:94:11:44:50:cf:17:70:8d:9c:14:c5:ed:94: + 4d:ba:94:9b:db:8b:9f:55:a1:5e:0a:90:bb:a0:0e:0d:3b:a0: + dd:4d:47:d1:cf:d0:47:6b:ff:6f:af:e4:83:40:73:e6:3a:59: + 40:bd:4f:3a:21:22:63:27:5d:02:26:67:89:1d:2f:19:c5:77: + e6:b5:67:4a:dd:b5:0e:de:46:34:57:c7:03:84:5d:cd:34:08: + 8f:47:c9:4d:7f:04:c0:4f:ff:68:52:bb:ae:84:0e:54:ce:5c: + 27:71:0f:a2:3f:9f:7a:49:a8:fa:0a:45:cf:96:42:a7:65:23: + b9:3e:40:eb:46:7b -----BEGIN CERTIFICATE----- -MIIHDTCCBXWgAwIBAgIJAMstgJlaaVJfMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +MIIHDzCCBXegAwIBAgIJAMstgJlaaVJfMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW -MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODA4MjkxNDIzMTZaFw0zNzEwMjgx -NDIzMTZaMF0xCzAJBgNVBAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEj -MCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xEDAOBgNVBAMMB2Fs -bHNhbnMwggGiMA0GCSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQCczEVv5D2UDtn6 -DMmZ/uCWCLyL+K5xTZp/5j3cyISoaTuU1Ku3kD97eLgpHj4Fgk5ZJi21zsQqepCj -jAhBk6tj6RYUcnMbb8MuxUkQMEDW+5LfSyp+HCaetlHosWdhEDqX4kpJ5ajBwNRt -07mxQExtC4kcno0ut9rG5XzLN29XpCpRHlFFrntOgQAEoiz9/fc8qaTgb37RgGYP -Qsxh7PcRDRe4ZGx1l06Irr8Y+2W50zWCfkwCS3DaLDOKIjSOfPHNqmfcfsTpzrj8 -330cdPklrMIuiBv+iGklCjkPZJiEhxvY2k6ERM4HAxxuPCivrH5MCeMNYvBVUcLr -GROm7JRXRllI/XubwwoAaAb+y+dZtCZ9AnzHIb+nyKiJxWAjzjR+QPL6jHrVWBVA -WTc83YP5FvxUXMfY3sVv9tNSCV3cpYOW5+iXcQzLuczXnOLRYk7p9wkb0/hk9KuK -4BMA90eBhvFMCFgHJ1/xJg2nFmBHPo/xbcwPG/ma5T/McA8mAlECAwEAAaOCAt4w -ggLaMIIBMAYDVR0RBIIBJzCCASOCB2FsbHNhbnOgHgYDKgMEoBcMFXNvbWUgb3Ro -ZXIgaWRlbnRpZmllcqA1BgYrBgEFAgKgKzApoBAbDktFUkJFUk9TLlJFQUxNoRUw -E6ADAgEBoQwwChsIdXNlcm5hbWWBEHVzZXJAZXhhbXBsZS5vcmeCD3d3dy5leGFt -cGxlLm9yZ6RnMGUxCzAJBgNVBAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJh -eDEjMCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xGDAWBgNVBAMM -D2Rpcm5hbWUgZXhhbXBsZYYXaHR0cHM6Ly93d3cucHl0aG9uLm9yZy+HBH8AAAGH -EAAAAAAAAAAAAAAAAAAAAAGIBCoDBAUwDgYDVR0PAQH/BAQDAgWgMB0GA1UdJQQW -MBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMBAf8EAjAAMB0GA1UdDgQWBBQx -XsBeL0f/i5L57j2xh9BTdTuxSDB9BgNVHSMEdjB0gBTz7JSO8o4wxI5owr+OahnA -wZ92ZaFRpE8wTTELMAkGA1UEBhMCWFkxJjAkBgNVBAoMHVB5dGhvbiBTb2Z0d2Fy -ZSBGb3VuZGF0aW9uIENBMRYwFAYDVQQDDA1vdXItY2Etc2VydmVyggkAyy2AmVpp -UlswgYMGCCsGAQUFBwEBBHcwdTA8BggrBgEFBQcwAoYwaHR0cDovL3Rlc3RjYS5w -eXRob250ZXN0Lm5ldC90ZXN0Y2EvcHljYWNlcnQuY2VyMDUGCCsGAQUFBzABhilo -dHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rlc3RjYS9vY3NwLzBDBgNVHR8E -PDA6MDigNqA0hjJodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rlc3RjYS9y -ZXZvY2F0aW9uLmNybDANBgkqhkiG9w0BAQsFAAOCAYEAckKm/O48IUcFM+iMaycH -Su3igUeWeUP/D+9aBqpMAXBbIcS3XRcpyBACwwh7jIZWnul8bqi2JhOeHh+TZoVn -Y54I+1U5VoL1vgw4HuvEVLKnexhVuwCHQ1BQu+EpEM89yQfH0l22RWgf1t4Alj4p -c/YicCGiumgolOw3vKcAcFhO0Uiu740RpG4QL5KDB+J2rL9Pu9afR56kAgMW+KgK -PWcXMUQOaNDTJNXnv2cwj4iXkgoe13TffntMxtnDhJIroIkRCEzdMknfNiPUY1bk -8Whab6DDPOI27vNGYHhNdqVaSmHG+K4YaMKNDi92ULu+uVbxBFysrdfWpB5FRVL0 -EKIPm+PZcxe2UkKmW8npjWB0aNAfes4Bjp5Vy89kwcyacqq0X7VVE0EQUaAspVtD -EsrMt8Ss8m9y/Q1QataBwZGTIf7emr7lPCqYlaFC+PJcdcbx/RGxIiYzW0NjIQZh -0s0E8zDGqD8X0wWjh0UuUh5RiONZTHhRsHu0WNknIm6M +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAgFw0xODA4MjkxNDIzMTZaGA8yNTI1MTAy +ODE0MjMxNlowXTELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4 +MSMwIQYDVQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEQMA4GA1UEAwwH +YWxsc2FuczCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAJgRj6UjBL/D +6ABTk9/FJ4viowXlO0WSuJ8vShBoKNYVvyN5fZFvW0po5vKtiZNRFVZsyQjUsk5B +Zi0aUA2jaa9/hUPXBln/H9ObKXF7cDuK1JNmBm4Zcse0U/3CTvJLjoGVVw2rdmg6 +LuSTw3FOJSqCyYuDc2P4h+Tw4jtzUFw/alrXBk7+Xc9Naq0VIuwUS84rk3pk+ogs +BSLm8SQ6IV5/FUce8Pg3chZIWBLCIOL0qGe11HBr5rGxnGOr+jvQWNA7mrlC7Kb1 +oOqqjjs8zKQRvWB1MFArWY+WIwqwuLnPJwj/KmbU8Rwfnf738X5TYHVua8D/08HH +xPuFTJyUFtAcKZ1xchgKAWvkBsZLeMV5U+IXuEI+PWT+NylrJdkC1aaolOWRcoZs +zrwCAz8Mc7R8YJgZFTDa+FXdkhEKbQGrB+it4WH7agzJmMOO+j+5mNk1rN2lzC87 +LRyUWQ5HBUjTNMiLFdhJXjE3UFfzbUkK6Fi5uujz2b4xKPE7XiqCzQIDAQABo4IC +3jCCAtowggEwBgNVHREEggEnMIIBI4IHYWxsc2Fuc6AeBgMqAwSgFwwVc29tZSBv +dGhlciBpZGVudGlmaWVyoDUGBisGAQUCAqArMCmgEBsOS0VSQkVST1MuUkVBTE2h +FTAToAMCAQGhDDAKGwh1c2VybmFtZYEQdXNlckBleGFtcGxlLm9yZ4IPd3d3LmV4 +YW1wbGUub3JnpGcwZTELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRo +cmF4MSMwIQYDVQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEYMBYGA1UE +AwwPZGlybmFtZSBleGFtcGxlhhdodHRwczovL3d3dy5weXRob24ub3JnL4cEfwAA +AYcQAAAAAAAAAAAAAAAAAAAAAYgEKgMEBTAOBgNVHQ8BAf8EBAMCBaAwHQYDVR0l +BBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYE +FHMNTh9K7PJTD1P8hW/Pgs2j6BGNMH0GA1UdIwR2MHSAFMAKKyhD3l/JfUflR5s2 +8mWMZzvioVGkTzBNMQswCQYDVQQGEwJYWTEmMCQGA1UECgwdUHl0aG9uIFNvZnR3 +YXJlIEZvdW5kYXRpb24gQ0ExFjAUBgNVBAMMDW91ci1jYS1zZXJ2ZXKCCQDLLYCZ +WmlSWzCBgwYIKwYBBQUHAQEEdzB1MDwGCCsGAQUFBzAChjBodHRwOi8vdGVzdGNh +LnB5dGhvbnRlc3QubmV0L3Rlc3RjYS9weWNhY2VydC5jZXIwNQYIKwYBBQUHMAGG +KWh0dHA6Ly90ZXN0Y2EucHl0aG9udGVzdC5uZXQvdGVzdGNhL29jc3AvMEMGA1Ud +HwQ8MDowOKA2oDSGMmh0dHA6Ly90ZXN0Y2EucHl0aG9udGVzdC5uZXQvdGVzdGNh +L3Jldm9jYXRpb24uY3JsMA0GCSqGSIb3DQEBCwUAA4IBgQClPRKHuTs+nBxZ+9U4 +Iklh88MRU0tOY6/yPe8kZ0W8dFxKZcW0u/6EuLbKffyq/4CuZx/Lw82P+XWM+dM/ +2/aB2AZCw12pHqOBfVesl9m9yM4e7HTXlNXfsa3OhBRcjEWkqOtnqxZXYRWGrhEe +tRBC3oR2iJs3Eqqmd0J1tMAEs3VF4NeqNOMHxe34TvA5mR9b2E4MrWRtCQc/4+Gf +G2UHllmatfFNw+z3MqEFlNELGFQ8Z8849Svsy715vvcbt3E/xkSAfwDcPaAHwLUc ++1L2oPiSxsZzB8XKCwR8VVHjupMyF71hrs8T5F4DslERx2j1CLYOV0kRPOP0DuGW +IEQoMZQRRFDPF3CNnBTF7ZRNupSb24ufVaFeCpC7oA4NO6DdTUfRz9BHa/9vr+SD +QHPmOllAvU86ISJjJ10CJmeJHS8ZxXfmtWdK3bUO3kY0V8cDhF3NNAiPR8lNfwTA +T/9oUruuhA5UzlwncQ+iP596Saj6CkXPlkKnZSO5PkDrRns= -----END CERTIFICATE----- diff --git a/Lib/test/certdata/capath/b1930218.0 b/Lib/test/certdata/capath/b1930218.0 index aa9dbbe294f..6d773ee5b32 100644 --- a/Lib/test/certdata/capath/b1930218.0 +++ b/Lib/test/certdata/capath/b1930218.0 @@ -1,27 +1,27 @@ -----BEGIN CERTIFICATE----- -MIIEgDCCAuigAwIBAgIJAMstgJlaaVJbMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +MIIEgjCCAuqgAwIBAgIJAMstgJlaaVJbMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW -MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODA4MjkxNDIzMTZaFw0zNzEwMjgx -NDIzMTZaME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUg -Rm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcjCCAaIwDQYJKoZI -hvcNAQEBBQADggGPADCCAYoCggGBANCgm7G5O3nuMS+4URwBde0JWUysyL9qCvh6 -CPAl4yV7avjE2KqgYAclsM9zcQVSaL8Gk64QYZa8s2mBGn0Z/CCGj5poG+3N4mxh -Z8dOVepDBiEb6bm+hF/C2uuJiOBCpkVJKtC5a4yTyUQ7yvw8lH/dcMWt2Es73B74 -VUu1J4b437CDz/cWN78TFzTUyVXtaxbJf60gTvAe2Ru/jbrNypbvHmnLUWZhSA3o -eaNZYdQQjeANOwuFttWFEt2lB8VL+iP6VDn3lwvJREceVnc8PBMBC2131hS6RPRT -NVbZPbk+NV/bM5pPWrk4RMkySf5m9h8al6rKTEr2uF5Af/sLHfhbodz4wC7QbUn1 -0kbUkFf+koE0ri04u6gXDOHlP+L3JgVUUPVksxxuRP9vqbQDlukOwojYclKQmcZB -D0aQWbg+b9Linh02gpXTWIoS8+LYDSBRI/CQLZo+fSaGsqfX+ShgA+N3x4gEyf6J -d3AQT8Ogijv0q0J74xSS2K4W1qHefQIDAQABo2MwYTAdBgNVHQ4EFgQU8+yUjvKO -MMSOaMK/jmoZwMGfdmUwHwYDVR0jBBgwFoAU8+yUjvKOMMSOaMK/jmoZwMGfdmUw -DwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAYYwDQYJKoZIhvcNAQELBQAD -ggGBAIsAVHKzjevzrzSf1mDq3oQ/jASPGaa+AmfEY8V040c3WYOUBvFFGegHL9ZO -S0+oPccHByeS9H5zT4syGZRGeiXE2cQnsBFjOmCLheFzTzQ7a6Q0jEmOzc9PsmUn -QRmw/IAxePJzapt9cTRQ/Hio2gW0nFs6mXprXe870+k7MwESZc9eB9gZr9VT6vAQ -rMS2Jjw0LnTuZN0dNnWJRACwDf0vswHMGosCzWzogILKv4LXAJ3YNhXSBzf8bHMd -2qgc6CCOMnr+bScW5Fhs6z7w/iRSKXG4lntTS0UgVUBehhvsyUaRku6sk2WRLpS2 -tqzoozSJpBoSDU1EpVLti5HuL6avpJUl+c7HW6cA05PKtDxdTfexPMxttEW+gu0Y -kMiG0XVRUARM6E/S1lCqdede/6F7Jxkca0ksbE1rY8w7cwDzmSbQgofTqTactD25 -SGiokvAnjgzNFXZChIDJP6N+tN3X+Kx2umCXPFofTt5x7gk5EN0x1WhXXRrlQroO -aOZF0w== +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAgFw0xODA4MjkxNDIzMTZaGA8yNTI1MTAy +ODE0MjMxNlowTTELMAkGA1UEBhMCWFkxJjAkBgNVBAoMHVB5dGhvbiBTb2Z0d2Fy +ZSBGb3VuZGF0aW9uIENBMRYwFAYDVQQDDA1vdXItY2Etc2VydmVyMIIBojANBgkq +hkiG9w0BAQEFAAOCAY8AMIIBigKCAYEA5rri5MHCDBw+Yti5XFcuUriDxYg65pp6 +9WQWM+s3bi9782gDRWVHXXEQWcorGwBsgRRh9IZZP+r9eDcWnUPxxPZpjMUpBoie +JiIErATYhzRIOetr8guSqsNuY1FRa8Kt/1zIL7IbnCCKQD6iL2rqyNk3Q1zc7ZLi +2UDSYZ9xivXtObqoXj61IWMQ2G+04hEBCxDou/ti70hVvPXSnKtorpUlGfKXfRrc +ZuqIXobky8tpTV6wo/tsMeQoYF6Q8dQuEFDhhfANXL3dRSQIGT4ck2aPK9pTfQQc +DkLEaF6mzakY7afNatDRhrqQ/7dM3sdDJG3HHGucgefhG1clkKkOyVbz9mteLbQu +QFCbQmPS1pkcONzPKyyncvHHXmM0dkj0PogTnoYWUy90+4cBjSKkaDPuE2x6BhRU +VhdXV5g00AtmCeOICfilFRwQc9CIUJleGGU7/zEnG17GqkH9LS8Yp8Dyq8citQtp +0nPRu9AcPfqkNWLNM4bHoCMPuWrV0m2NAgMBAAGjYzBhMB0GA1UdDgQWBBTACiso +Q95fyX1H5UebNvJljGc74jAfBgNVHSMEGDAWgBTACisoQ95fyX1H5UebNvJljGc7 +4jAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG9w0BAQsF +AAOCAYEAvONWIgPkXLln/5S8dZwAhbDVnMTDKWZei7KppjCGcRpr8gDFgqtfUAQq +++24TLkAG0lXkhHNorzLD6i0YfgUyqDsQBe6VaHEvKayWu/0IBB3R9CgxVi5bLUQ +e4VKQ6P7LAG5dxewvqDurq5NZ4ZIiVeGeOo87fBBNY1xaFX58umsMtTGou/sVObE +jiz9vapgVmUzleoQxnQE6ypumxH2YQCq/ezyC7FLEc2T69+Yrky0BwRK5e//Ulh1 +9T6kceFKclypj9SqiPBqcbTDAF+ZbteRr2yYDWTCJMeeBRFoXiRi4y5F7KM08qOd +TeUyGC90/BHxNlBPoEApaFxDTCNsXXLE7FJ269yyvB+mxAZmm1zHzMry0SVP3qUf +jeQMSbbPhUChuR/GxxkVB2M0k9BXoFpw7K9KHHIXHXSjbDFFCzN6obhG28cOZExv +t5kEgkMf4FnWmSEnKAlArvzEI6qgDAgFKpIc2yOe0dVjrjkToxKIWkM8Sm4y8ISf ++QkMkee4 -----END CERTIFICATE----- diff --git a/Lib/test/certdata/capath/ceff1710.0 b/Lib/test/certdata/capath/ceff1710.0 index aa9dbbe294f..6d773ee5b32 100644 --- a/Lib/test/certdata/capath/ceff1710.0 +++ b/Lib/test/certdata/capath/ceff1710.0 @@ -1,27 +1,27 @@ -----BEGIN CERTIFICATE----- -MIIEgDCCAuigAwIBAgIJAMstgJlaaVJbMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +MIIEgjCCAuqgAwIBAgIJAMstgJlaaVJbMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW -MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODA4MjkxNDIzMTZaFw0zNzEwMjgx -NDIzMTZaME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUg -Rm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcjCCAaIwDQYJKoZI -hvcNAQEBBQADggGPADCCAYoCggGBANCgm7G5O3nuMS+4URwBde0JWUysyL9qCvh6 -CPAl4yV7avjE2KqgYAclsM9zcQVSaL8Gk64QYZa8s2mBGn0Z/CCGj5poG+3N4mxh -Z8dOVepDBiEb6bm+hF/C2uuJiOBCpkVJKtC5a4yTyUQ7yvw8lH/dcMWt2Es73B74 -VUu1J4b437CDz/cWN78TFzTUyVXtaxbJf60gTvAe2Ru/jbrNypbvHmnLUWZhSA3o -eaNZYdQQjeANOwuFttWFEt2lB8VL+iP6VDn3lwvJREceVnc8PBMBC2131hS6RPRT -NVbZPbk+NV/bM5pPWrk4RMkySf5m9h8al6rKTEr2uF5Af/sLHfhbodz4wC7QbUn1 -0kbUkFf+koE0ri04u6gXDOHlP+L3JgVUUPVksxxuRP9vqbQDlukOwojYclKQmcZB -D0aQWbg+b9Linh02gpXTWIoS8+LYDSBRI/CQLZo+fSaGsqfX+ShgA+N3x4gEyf6J -d3AQT8Ogijv0q0J74xSS2K4W1qHefQIDAQABo2MwYTAdBgNVHQ4EFgQU8+yUjvKO -MMSOaMK/jmoZwMGfdmUwHwYDVR0jBBgwFoAU8+yUjvKOMMSOaMK/jmoZwMGfdmUw -DwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAYYwDQYJKoZIhvcNAQELBQAD -ggGBAIsAVHKzjevzrzSf1mDq3oQ/jASPGaa+AmfEY8V040c3WYOUBvFFGegHL9ZO -S0+oPccHByeS9H5zT4syGZRGeiXE2cQnsBFjOmCLheFzTzQ7a6Q0jEmOzc9PsmUn -QRmw/IAxePJzapt9cTRQ/Hio2gW0nFs6mXprXe870+k7MwESZc9eB9gZr9VT6vAQ -rMS2Jjw0LnTuZN0dNnWJRACwDf0vswHMGosCzWzogILKv4LXAJ3YNhXSBzf8bHMd -2qgc6CCOMnr+bScW5Fhs6z7w/iRSKXG4lntTS0UgVUBehhvsyUaRku6sk2WRLpS2 -tqzoozSJpBoSDU1EpVLti5HuL6avpJUl+c7HW6cA05PKtDxdTfexPMxttEW+gu0Y -kMiG0XVRUARM6E/S1lCqdede/6F7Jxkca0ksbE1rY8w7cwDzmSbQgofTqTactD25 -SGiokvAnjgzNFXZChIDJP6N+tN3X+Kx2umCXPFofTt5x7gk5EN0x1WhXXRrlQroO -aOZF0w== +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAgFw0xODA4MjkxNDIzMTZaGA8yNTI1MTAy +ODE0MjMxNlowTTELMAkGA1UEBhMCWFkxJjAkBgNVBAoMHVB5dGhvbiBTb2Z0d2Fy +ZSBGb3VuZGF0aW9uIENBMRYwFAYDVQQDDA1vdXItY2Etc2VydmVyMIIBojANBgkq +hkiG9w0BAQEFAAOCAY8AMIIBigKCAYEA5rri5MHCDBw+Yti5XFcuUriDxYg65pp6 +9WQWM+s3bi9782gDRWVHXXEQWcorGwBsgRRh9IZZP+r9eDcWnUPxxPZpjMUpBoie +JiIErATYhzRIOetr8guSqsNuY1FRa8Kt/1zIL7IbnCCKQD6iL2rqyNk3Q1zc7ZLi +2UDSYZ9xivXtObqoXj61IWMQ2G+04hEBCxDou/ti70hVvPXSnKtorpUlGfKXfRrc +ZuqIXobky8tpTV6wo/tsMeQoYF6Q8dQuEFDhhfANXL3dRSQIGT4ck2aPK9pTfQQc +DkLEaF6mzakY7afNatDRhrqQ/7dM3sdDJG3HHGucgefhG1clkKkOyVbz9mteLbQu +QFCbQmPS1pkcONzPKyyncvHHXmM0dkj0PogTnoYWUy90+4cBjSKkaDPuE2x6BhRU +VhdXV5g00AtmCeOICfilFRwQc9CIUJleGGU7/zEnG17GqkH9LS8Yp8Dyq8citQtp +0nPRu9AcPfqkNWLNM4bHoCMPuWrV0m2NAgMBAAGjYzBhMB0GA1UdDgQWBBTACiso +Q95fyX1H5UebNvJljGc74jAfBgNVHSMEGDAWgBTACisoQ95fyX1H5UebNvJljGc7 +4jAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG9w0BAQsF +AAOCAYEAvONWIgPkXLln/5S8dZwAhbDVnMTDKWZei7KppjCGcRpr8gDFgqtfUAQq +++24TLkAG0lXkhHNorzLD6i0YfgUyqDsQBe6VaHEvKayWu/0IBB3R9CgxVi5bLUQ +e4VKQ6P7LAG5dxewvqDurq5NZ4ZIiVeGeOo87fBBNY1xaFX58umsMtTGou/sVObE +jiz9vapgVmUzleoQxnQE6ypumxH2YQCq/ezyC7FLEc2T69+Yrky0BwRK5e//Ulh1 +9T6kceFKclypj9SqiPBqcbTDAF+ZbteRr2yYDWTCJMeeBRFoXiRi4y5F7KM08qOd +TeUyGC90/BHxNlBPoEApaFxDTCNsXXLE7FJ269yyvB+mxAZmm1zHzMry0SVP3qUf +jeQMSbbPhUChuR/GxxkVB2M0k9BXoFpw7K9KHHIXHXSjbDFFCzN6obhG28cOZExv +t5kEgkMf4FnWmSEnKAlArvzEI6qgDAgFKpIc2yOe0dVjrjkToxKIWkM8Sm4y8ISf ++QkMkee4 -----END CERTIFICATE----- diff --git a/Lib/test/certdata/cert3.pem b/Lib/test/certdata/cert3.pem index 4ab0f5ff133..a11dc614657 100644 --- a/Lib/test/certdata/cert3.pem +++ b/Lib/test/certdata/cert3.pem @@ -1,34 +1,34 @@ -----BEGIN CERTIFICATE----- -MIIF8TCCBFmgAwIBAgIJAMstgJlaaVJcMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +MIIF8zCCBFugAwIBAgIJAMstgJlaaVJcMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW -MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODA4MjkxNDIzMTZaFw0zNzEwMjgx -NDIzMTZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEj -MCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMMCWxv -Y2FsaG9zdDCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAKAqKHEL7aDt -3swl8hQF8VaK4zDGDRaF3E/IZTMwCN7FsQ4ejSiOe3E90f0phHCIpEpv2OebNenY -IpOGoFgkh62r/cthmnhu8Mn+FUIv17iOq7WX7B30OSqEpnr1voLX93XYkAq8LlMh -P79vsSCVhTwow3HZY7krEgl5WlfryOfj1i1TODSFPRCJePh66BsOTUvV/33GC+Qd -pVZVDGLowU1Ycmr/FdRvwT+F39Dehp03UFcxaX0/joPhH5gYpBB1kWTAQmxuqKMW -9ZZs6hrPtMXF/yfSrrXrzTdpct9paKR8RcufOcS8qju/ISK+1P/LXg2b5KJHedLo -TTIO3yCZ4d1odyuZBP7JDrI05gMJx95gz6sG685Qc+52MzLSTwr/Qg+MOjQoBy0o -8fRRVvIMEwoN0ZDb4uFEUuwZceUP1vTk/GGpNQt7ct4ropn6K4Zta3BUtovlLjZa -IIBhc1KETUqjRDvC6ACKmlcJ/5pY/dbH1lOux+IMFsh+djmaV90b3QIDAQABo4IB -wDCCAbwwFAYDVR0RBA0wC4IJbG9jYWxob3N0MA4GA1UdDwEB/wQEAwIFoDAdBgNV -HSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4E -FgQUP7HpT6C+MGY+ChjID0caTzRqD0IwfQYDVR0jBHYwdIAU8+yUjvKOMMSOaMK/ -jmoZwMGfdmWhUaRPME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29m -dHdhcmUgRm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcoIJAMst -gJlaaVJbMIGDBggrBgEFBQcBAQR3MHUwPAYIKwYBBQUHMAKGMGh0dHA6Ly90ZXN0 -Y2EucHl0aG9udGVzdC5uZXQvdGVzdGNhL3B5Y2FjZXJ0LmNlcjA1BggrBgEFBQcw -AYYpaHR0cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0Y2Evb2NzcC8wQwYD -VR0fBDwwOjA4oDagNIYyaHR0cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0 -Y2EvcmV2b2NhdGlvbi5jcmwwDQYJKoZIhvcNAQELBQADggGBAMo0usXQzycxMtYN -JzC42xfftzmnu7E7hsQx/fur22MazJCruU6rNEkMXow+cKOnay+nmiV7AVoYlkh2 -+DZ4dPq8fWh/5cqmnXvccr2jJVEXaOjp1wKGLH0WfLXcRLIK4/fJM6NRNoO81HDN -hJGfBrot0gUKZcPZVQmouAlpu5OGwrfCkHR8v/BdvA5jE4zr+g/x+uUScE0M64wu -okJCAAQP/PkfQZxjePBmk7KPLuiTHFDLLX+2uldvUmLXOQsJgqumU03MBT4Z8NTA -zqmtEM65ceSP8lo8Zbrcy+AEkCulFaZ92tyjtbe8oN4wTmTLFw06oFLSZzuiOgDV -OaphdVKf/pvA6KBpr6izox0KQFIE5z3AAJZfKzMGDDD20xhy7jjQZNMAhjfsT+k4 -SeYB/6KafNxq08uoulj7w4Z4R/EGpkXnU96ZHYHmvGN0RnxwI1cpYHCazG8AjsK/ -anN9brBi5twTGrn+D8LRBqF5Yn+2MKkD0EdXJdtIENHP+32sPQ== +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAgFw0xODA4MjkxNDIzMTZaGA8yNTI1MTAy +ODE0MjMxNlowXzELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4 +MSMwIQYDVQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjESMBAGA1UEAwwJ +bG9jYWxob3N0MIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEA7jZwqmFq +ILsyiAdz8DrKEcyPMIDXccd0YiCUiLRSEpQIAeF5GMzwaYuZxUfBgqquyFEEPomM +HWWt8h+l9dSHZawseIA3UBUhTnoJxuNaKr+xsARBb6usZaMKaGhsPuf/P7CV/1VO +OKy/f34jFU23oTITEv8+Z00mEgAle7EV58FuE+pdjne+xczwY52hRQza+RiKIg+J +jUid+bdObZYhnM9CMhOUxkepCBBTSB+bYXh6CSeCQuLi8licHiacQ8ddJ41kcCjf +7V5vBZx0DzEQFJdsDNO0GRCNcn81K9NP6BtnaT5z8jYfuqdpXfCUtINvz3dqUC/D +rZjNnA3DeRPqghFtVFSCef/2nfKVHKEMMkSAUTiW2pKr+hXFU3YE6IKKuVbvk+k1 +RS0iEr1b6bFdDLU/x/f/U7Qp6jsJYhPLPJG9zY0E/Hu9lRzXeN21TxOA3kPl5WzK +Cs1fhjpkh0n80jmQfZEnEphneWA/O/N02y/P+zZ9REUHVqmosRiN+vgRAgMBAAGj +ggHAMIIBvDAUBgNVHREEDTALgglsb2NhbGhvc3QwDgYDVR0PAQH/BAQDAgWgMB0G +A1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMBAf8EAjAAMB0GA1Ud +DgQWBBQWIsmqINT0ju2cprsj9fIpRO3yHjB9BgNVHSMEdjB0gBTACisoQ95fyX1H +5UebNvJljGc74qFRpE8wTTELMAkGA1UEBhMCWFkxJjAkBgNVBAoMHVB5dGhvbiBT +b2Z0d2FyZSBGb3VuZGF0aW9uIENBMRYwFAYDVQQDDA1vdXItY2Etc2VydmVyggkA +yy2AmVppUlswgYMGCCsGAQUFBwEBBHcwdTA8BggrBgEFBQcwAoYwaHR0cDovL3Rl +c3RjYS5weXRob250ZXN0Lm5ldC90ZXN0Y2EvcHljYWNlcnQuY2VyMDUGCCsGAQUF +BzABhilodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rlc3RjYS9vY3NwLzBD +BgNVHR8EPDA6MDigNqA0hjJodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rl +c3RjYS9yZXZvY2F0aW9uLmNybDANBgkqhkiG9w0BAQsFAAOCAYEAQ4IfGLTLerdO +rMNlLrdXOvB4s7IgPr17JPfnF8xiwLhj8C4wDFS+yZR8VNRABm6SnXIsRPXjwUo/ +JuQhyrrvT6NQVu6JXNxbmLwM6dsWmPBMP2W52eAuvYOexxv3T4dfdf9nXQr/zvoL +8dWLWCMrkie4Ff9mwvlo4u1koErgQousNWpnZPXLqQA3IFbdOgJu2A+0Xf+Sow1l +/C6rTje8ftZbHFV4oG6pLlUxz2HwG0z+/mB1dujZofUU8EMzTVIFvjP/2jGUvQ3l +Taju0fOSNMI2kTc6bewg37Oeol3Q8KHi/7eFzgnjyEpqk6Su7MFnQveOL2TK13Zy +vz/vZP8Q3aI+LfWqAs8x8G2Ta1ZMsIiVVNzUrNzBiCeL2ZxOZpP43V50QSaa7+jI +RlzV9PzNzGfHM2IucJvROd40/a2duUhh54lTYmLwQGxoL+HaQGEqUK/JQW2YFq/L +YwPsBJngOZhgrqpqV5slcwMWv3jI1y/r/GR/x3iMNBVbZkCYhuYK -----END CERTIFICATE----- diff --git a/Lib/test/certdata/idnsans.pem b/Lib/test/certdata/idnsans.pem index 07a42422af1..ebd7250cb4e 100644 --- a/Lib/test/certdata/idnsans.pem +++ b/Lib/test/certdata/idnsans.pem @@ -1,42 +1,42 @@ -----BEGIN PRIVATE KEY----- -MIIG/AIBADANBgkqhkiG9w0BAQEFAASCBuYwggbiAgEAAoIBgQCp6zt40WB3K7yj -BGugnRuqI3ApftThZWDIpvW0cVmN0nqQxsO6CCnS4dS7SYhGFiIqWjNVc2WG0gv7 -nC5DFguqbndNZk9/SjX8EOxKz4ANjd61WnTkDO5Tbiiyd+TuEBxhmbEF69bF9dtd -1Sgo8jmM7j+aa6ClYh/49bx+blJDF76EGSrmB1q+obMeZURhPXNBeoiqKR83x5Hc -LTJYMocvb6m8uABwuSka13Gb3QGu06p5ldK6TDK38HsoOy6MFO5F1PrkakG/eBHO -jcBOGPfNmTwWOqvwlcQWykr4QspWS+yTzdkgZ+mxar/yQuq7wuYSNaEfGH5yoYtV -WIgKwwZRDPqpSQuVe+J+MWLPQ6RTM+rXIHVzHtPk1f8DrgN+hSepJy/sVBBEQCzj -nyB+scn76ETWch3iyVoMj3oVOGs0b4XTDMmUw/DmEt5TDah7TqE3G+fpBIbgMSjx -MzUQZl27izmM9nQCJRAosNoNwXqlM754K9WcY6gT8kkcj1CfTmMCAwEAAQKCAYAz -9ZdHkDsf5fN2pAznXfOOOOz8+2hMjmwkn42GAp1gdWr+Z5GFiyaC8oTTSp6N1AnZ -iqCk8jcrHYMFi1JIOG8TzFjWBcGsinxsmp4vGDmvq2Ddcw5IiD2+rHJsdKZAOBP9 -snpD9cTE3zQYAu0XbE617krrxRqoSBO/1SExRjoIgzPCgFGyarBQl/DGjC/3Tku2 -y6oL4qxFqdTMD9QTzUuycUJlz5xu2+gaaaQ3hcMUe2xnZq28Qz3FKpf2ivZmZqWf -4+AIe0lRosmFoLAFjIyyuGCkWZ2t9KDIZV0OOS4+DvVOC/Um9r4VojeikripCGKY -2FzkkuQP3jz6pJ1UxCDg7YXZdR2IbcS18F1OYmLViU8oLDR6T01s0Npmp39dDazf -A4U+WyV3o1ydiSpwAiN8pJadmr5BHrCSmawV8ztW5yholufrO+FR5ze2+QG99txm -6l7lUI8Nz01lYG6D10MjaQ9INk2WSjBPVNbfsTl73/hR76/319ctfOINRMHilJ0C -gcEAvFgTdc5Qf9E7xEZE5UtpSTPvZ24DxQ7hmGW3pTs6+Nw4ccknnW0lLkWvY4Sf -gXl4TyTTUhe6PAN3UHzbjN2vdsTmAVUlnkNH40ymF8FgRGFNDuvuCZvf5ZWNddSl -6Vu/e5TFPDAqS8rGnl23DgXhlT+3Y0/mrftWoigdOxgximLAsmmqp/tANGi9jqE1 -3L0BN+xKqMMKSXkMtuzJHuini8MaWQgQcW/4czh4ArdesMzuVrstOD8947XHreY9 -pesVAoHBAOb0y/AFEdR+mhk/cfqjTzsNU2sS9yHlzMVgecF8BA26xcfAwg4d47VS -+LK8fV6KC4hHk4uQWjQzCG2PYXBbFT52xeJ3EC8DwWxJP09b4HV/1mWxXl5htjnr -dfyTmXKvEe5ZBpKGWc8i7s7jBi7R5EpgIfc586iNRyjYAk60dyG0iP13SurRvXBg -ID25VR4wABl3HQ3Hhv61dqC9FPrdHZQJdysfUqNrAFniWsSR2eyG5i4S1uHa3G+i -MzBTOuBRlwKBwBNXUBhG6YlWqTaMqMKLLfKwfKM4bvargost1uAG5xVrN/inWYQX -EzxfN5WWpvKa0Ln/5BuICD3ldTk0uS8MDNq7eYslfUl1S0qSMnQ6DXK4MzuXCsi9 -0w42f2JcRfVi0JUWP/LgV1eVKTRWF1g/Tl0PP/vY1q2DI/BfAjFxWJUHcxZfN4Es -kflP0Dd3YpqaZieiAkC2VrYY0i9uvXCJH7uAe5Is+9NKVk8uu1Q8FGM/iDIr4obm -J6rcnfbDsAz7yQKBwGtIbW9qO3UU9ioiQaTmtYg90XEclzXk1HEfNo+9NvjVuMfo -b3w1QDBbgXEtg6MlxuOgNBaRkIVM625ROzcA6GZir9tZ6Wede/z8LW+Ew0hxgLsu -YCLBiu9uxBj2y0HttwubySTJSfChToNGC/o1v7EY5M492kSCk/qSFMhQpkI+5Z+w -CVn44eHQlUl2zOY/79vka9eZxsiMrLVP/+3kRrgciYG7hByrOLeIIRfMlIl9xHDE -iZLSorEsjFC3aNMIswKBwFELC2fvlziW9rECQcGXnvc1DPmZcxm1ATFZ93FpKleF -TjLIWSdst0PmO8CSIuEZ2ZXPoK9CMJyQG+kt3k7IgZ1xKXg9y6ThwbznurXp1jaW -NjEnYtFMBK9Ur3oaAsrG2XwZ2PMvnI/Yp8tciGvjJlzSM8gHJ9BL8Yf+3gIJi/0D -KtaF9ha9J/SDDZdEiLIQ4LvSqYmlUgsCgiUvY3SVwCh8xDfBWD1hKw9vUiZu5cnJ -81hAHFgeD4f+C8fLols/sA== +MIIG/QIBADANBgkqhkiG9w0BAQEFAASCBucwggbjAgEAAoIBgQDeGGysqNqsz64+ +xsN+CUZDfCgPPhzUK0lDUlkEkp0ZeFeQm3+po2Y82gvJIValO5ChiXjPxSgJwoto +SHe4QZ8fWAlkX1d19A8xjjetx1K3dqZstqe3mPP7HQvfJlRkmAtVhFaUj1+SUx9K +s4HB4Lk2hqKUoq3TlRMpeSfJTXaHThVP7KZPWBzfM7pdpGYnznBpxgfymNrpTcXN +CjeR6lUlj/TExHiYLj4E0aODR/SUz0tBNdCQRcRDw+OyREe6Rk32Krl34Pr1dNKe +ER0tulhM2wRFsqw0a8vNs13I30BH4aXAr5bGgLCF2tXbP3KoeoR/GkQSWV14KrhT +xhKD3QEJ8gHjoHYKr4IdcFYcTakbI9E3THsQjQTUMJ/Xygza7OOr6pNPKY10tYfp +7WhyyY7cB/C/rHnUY0f58IZlRdMYgcLCUl7wzUxSVEMyr9Ary8XZZlxuO629k2ud +AIGwIi2TVq+UK0kNPZyGicGHMeEhoNc3YDMB/zYeqMr4KryI+mkCAwEAAQKCAYA4 +EENSnHdC+1P5bdRIew/dFjjIjD3bwyeF0oI9GMOGe+3ix5YE3QYAY2xpM7y7Dhu2 +40x3akXunMjzJKPwA8SmtWL9juG1mUvCjyt39yJmxJFDTSJuQrKIF694/6R7FjR6 +PGNcsgqGlewGv+SH6/HlFTxyN9SYXf/NztMfyimbAzd3Cv56dfwnzdeELu1IrCCN +WtuDvlk4XpUJasRXVadzyXCYwR3OEJJARik4CRBxBhjxl6OT38Co+IiAZiMTHw6z +jXeRuvyxXyYB76BQs9uCt05c5JV1I2OT1aKdpC0EbHlJEzJhmxrpdfBeoVdJUVVg +4b0QoocZE89KXOVmjmIKExxmjOjxH6o6qfxvDQWLspADD4D+69Zqjj6hFAfn3EUW +hcq3RHwuUheCE6i2xRtWPnA1ygqWN9mdNojX1+EXIv0Dh29kFWHpMUC29WWm9JA6 +Arx2UrtvoWGYotKxTDxZ0SV5ws/LzGjvMaT/dbXZHjywnB7sPfpcE9b91p8Q6VcC +gcEA+Wr44ZzyMirF9FYYA+KEBL91khmffhXzCDkRXqd1j2BCQWETWqvrUuAx+kwh +FZ5AABdip96w8Eaj8XBWb1c81mJUbOimbcqiV6B4GokCI/7u3FkOMPFRNBq41R/M +43JH6QLHNxhXI4OlZsxv5WHmR5N88kXEHhofBoSn14EHtdkK/Hu8ly1Wux4/da83 +2wanwlysmQkk1CnpJVT0st7HrKHP8YWDbcBvH+xysHOIOs22q3gvAF2rsdORidpR +fZnHAoHBAOP03brYpLU1xa5LHQz/B7uWfgJm3l7wRQ627YOfUBe4IjnH2y/kNKjv +ROVjaooB2gtya6Z6xVuY1m4LzUfYuRN1FdX1Np4NjvfgPNFViX+0HoxyAytXn++V +Iy1tCpL6X43Iqrj/VxBcX09vIpbsprW2d3dcIo/45abkAO6Uoa6HUmZoqrqrBA7b +LnuhEfBAyyN42OiXbHDqif7XAWC30yY6k0FfDaOSLiBkAOKZ9lkf7ei3eNr4SdOa +Tf5a/RVKTwKBwB7YfOkiCM3tfkfGcffhBqSzrO2hn5jvS/wjWqOTIDXYGLmPMN6Q +zmyUb3nd+mV7Cb05JylNoCJHCjVsyDPC3TJCPOCvMQ349nTR0qitcwdSmuXDWb7x +yTIhb+Rjp2olkwEdJ9gHeZdZy5XYCKqcnecSNWyc9jEm19lthHhha7uwmOw6vUsQ +/13q0rxSLB05SHwADBRtDhHzEPNd+1k3tggChv3+ng9vsg6HpnNuBlYHZOT12xI3 +g2ldme0rg9J9twKBwA4R1gGrT3czy3C3iCJ+Ny732e0yBjWb5NdEqSI/mgTsw4gH +ctrg3fMzWXBDE5dTB+8+77AF0dqWc121csUldj7iMifTi7xzn8hi2b4d5m+wYVZP +zyxEq0VxUguCuG1b8Lvij879S5Vh7iwL8vmXv65lhbgjQqraNOp5FimjmNsZ1Rcn +DKqa1ZRJKPROe7n1ddRJqDGq7vGFOGE3Sgl7Lxgj82TMhh37bsdnBLr3v8G+e8Oq +V1ZEjuH1myzA1vASdwKBwQDcMHbUKeoJUxyIlABFsWxhm0MKwL5ZRgo7rhH01rF2 +TF3mbAEHHsAEfkHgBVRooifbxglxUDy1olIyRk+kqs1gmXZA0UmE9lUqoX5+n/j8 +pgbCc3sV53x7JJ4BCnAn49XTOnTi2ILeQ3MvTqcj+sDKo+0T4Zq0r5d8ZrMikKbO +HJ4MMBGth645QKhgpb0XgltvSn8aceS4uTxIKrbNpZVnWj/VzuOYoXQmKnZko9p1 +dbyjt6PMeVXWj0tz8FB2DGE= -----END PRIVATE KEY----- Certificate: Data: @@ -47,38 +47,38 @@ Certificate: Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server Validity Not Before: Aug 29 14:23:16 2018 GMT - Not After : Oct 28 14:23:16 2037 GMT + Not After : Oct 28 14:23:16 2525 GMT Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=idnsans Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (3072 bit) Modulus: - 00:a9:eb:3b:78:d1:60:77:2b:bc:a3:04:6b:a0:9d: - 1b:aa:23:70:29:7e:d4:e1:65:60:c8:a6:f5:b4:71: - 59:8d:d2:7a:90:c6:c3:ba:08:29:d2:e1:d4:bb:49: - 88:46:16:22:2a:5a:33:55:73:65:86:d2:0b:fb:9c: - 2e:43:16:0b:aa:6e:77:4d:66:4f:7f:4a:35:fc:10: - ec:4a:cf:80:0d:8d:de:b5:5a:74:e4:0c:ee:53:6e: - 28:b2:77:e4:ee:10:1c:61:99:b1:05:eb:d6:c5:f5: - db:5d:d5:28:28:f2:39:8c:ee:3f:9a:6b:a0:a5:62: - 1f:f8:f5:bc:7e:6e:52:43:17:be:84:19:2a:e6:07: - 5a:be:a1:b3:1e:65:44:61:3d:73:41:7a:88:aa:29: - 1f:37:c7:91:dc:2d:32:58:32:87:2f:6f:a9:bc:b8: - 00:70:b9:29:1a:d7:71:9b:dd:01:ae:d3:aa:79:95: - d2:ba:4c:32:b7:f0:7b:28:3b:2e:8c:14:ee:45:d4: - fa:e4:6a:41:bf:78:11:ce:8d:c0:4e:18:f7:cd:99: - 3c:16:3a:ab:f0:95:c4:16:ca:4a:f8:42:ca:56:4b: - ec:93:cd:d9:20:67:e9:b1:6a:bf:f2:42:ea:bb:c2: - e6:12:35:a1:1f:18:7e:72:a1:8b:55:58:88:0a:c3: - 06:51:0c:fa:a9:49:0b:95:7b:e2:7e:31:62:cf:43: - a4:53:33:ea:d7:20:75:73:1e:d3:e4:d5:ff:03:ae: - 03:7e:85:27:a9:27:2f:ec:54:10:44:40:2c:e3:9f: - 20:7e:b1:c9:fb:e8:44:d6:72:1d:e2:c9:5a:0c:8f: - 7a:15:38:6b:34:6f:85:d3:0c:c9:94:c3:f0:e6:12: - de:53:0d:a8:7b:4e:a1:37:1b:e7:e9:04:86:e0:31: - 28:f1:33:35:10:66:5d:bb:8b:39:8c:f6:74:02:25: - 10:28:b0:da:0d:c1:7a:a5:33:be:78:2b:d5:9c:63: - a8:13:f2:49:1c:8f:50:9f:4e:63 + 00:de:18:6c:ac:a8:da:ac:cf:ae:3e:c6:c3:7e:09: + 46:43:7c:28:0f:3e:1c:d4:2b:49:43:52:59:04:92: + 9d:19:78:57:90:9b:7f:a9:a3:66:3c:da:0b:c9:21: + 56:a5:3b:90:a1:89:78:cf:c5:28:09:c2:8b:68:48: + 77:b8:41:9f:1f:58:09:64:5f:57:75:f4:0f:31:8e: + 37:ad:c7:52:b7:76:a6:6c:b6:a7:b7:98:f3:fb:1d: + 0b:df:26:54:64:98:0b:55:84:56:94:8f:5f:92:53: + 1f:4a:b3:81:c1:e0:b9:36:86:a2:94:a2:ad:d3:95: + 13:29:79:27:c9:4d:76:87:4e:15:4f:ec:a6:4f:58: + 1c:df:33:ba:5d:a4:66:27:ce:70:69:c6:07:f2:98: + da:e9:4d:c5:cd:0a:37:91:ea:55:25:8f:f4:c4:c4: + 78:98:2e:3e:04:d1:a3:83:47:f4:94:cf:4b:41:35: + d0:90:45:c4:43:c3:e3:b2:44:47:ba:46:4d:f6:2a: + b9:77:e0:fa:f5:74:d2:9e:11:1d:2d:ba:58:4c:db: + 04:45:b2:ac:34:6b:cb:cd:b3:5d:c8:df:40:47:e1: + a5:c0:af:96:c6:80:b0:85:da:d5:db:3f:72:a8:7a: + 84:7f:1a:44:12:59:5d:78:2a:b8:53:c6:12:83:dd: + 01:09:f2:01:e3:a0:76:0a:af:82:1d:70:56:1c:4d: + a9:1b:23:d1:37:4c:7b:10:8d:04:d4:30:9f:d7:ca: + 0c:da:ec:e3:ab:ea:93:4f:29:8d:74:b5:87:e9:ed: + 68:72:c9:8e:dc:07:f0:bf:ac:79:d4:63:47:f9:f0: + 86:65:45:d3:18:81:c2:c2:52:5e:f0:cd:4c:52:54: + 43:32:af:d0:2b:cb:c5:d9:66:5c:6e:3b:ad:bd:93: + 6b:9d:00:81:b0:22:2d:93:56:af:94:2b:49:0d:3d: + 9c:86:89:c1:87:31:e1:21:a0:d7:37:60:33:01:ff: + 36:1e:a8:ca:f8:2a:bc:88:fa:69 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Alternative Name: @@ -90,9 +90,9 @@ Certificate: X509v3 Basic Constraints: critical CA:FALSE X509v3 Subject Key Identifier: - 5B:93:42:58:B0:B4:18:CC:41:4C:15:EB:42:33:66:77:4C:71:2F:42 + C8:06:99:B7:E8:8F:EC:4F:3D:5C:89:6A:06:F5:77:2E:B0:E0:6A:9E X509v3 Authority Key Identifier: - keyid:F3:EC:94:8E:F2:8E:30:C4:8E:68:C2:BF:8E:6A:19:C0:C1:9F:76:65 + keyid:C0:0A:2B:28:43:DE:5F:C9:7D:47:E5:47:9B:36:F2:65:8C:67:3B:E2 DirName:/C=XY/O=Python Software Foundation CA/CN=our-ca-server serial:CB:2D:80:99:5A:69:52:5B Authority Information Access: @@ -103,64 +103,64 @@ Certificate: URI:http://testca.pythontest.net/testca/revocation.crl Signature Algorithm: sha256WithRSAEncryption Signature Value: - 5f:d8:9b:dc:22:55:80:47:e1:9b:04:3e:46:53:9b:e5:a7:4a: - 8f:eb:53:01:39:d5:04:f6:cf:dc:48:84:8a:a9:c3:a5:35:22: - 2f:ab:74:77:ec:a6:fd:b1:e6:e6:74:82:38:54:0b:27:36:e6: - ec:3d:fe:92:1a:b2:7a:35:0d:a3:e5:7c:ff:e5:5b:1a:28:4b: - 29:1f:99:1b:3e:11:e9:e2:e0:d7:da:06:4f:e3:7b:8c:ad:30: - f4:39:24:e8:ad:2a:0e:71:74:ab:ed:62:e9:9f:85:7e:6a:b0: - bb:53:b4:d7:6b:b8:da:54:15:5c:9a:41:cf:61:f1:ab:67:d6: - 27:5c:0c:a3:d7:41:e7:27:3e:58:89:d6:1f:3f:2a:52:cc:13: - 0b:4b:e6:d6:ba:a0:c7:fd:e3:17:a4:b8:da:cc:cb:88:70:21: - 3b:70:df:09:40:6c:e7:02:81:08:80:b0:36:77:fb:44:c5:cf: - bf:19:54:7c:d1:4e:1f:a2:44:9e:d8:56:0e:bf:4b:0b:e0:84: - 6f:bc:f6:c6:7f:35:7a:17:ca:83:b3:82:c6:4e:d3:f3:d8:30: - 05:fd:6d:3c:8a:ab:63:55:6f:c5:18:ba:66:fe:e2:35:04:2b: - ae:76:34:f0:56:18:e8:54:db:83:b2:1b:93:0a:25:81:81:f0: - 25:ca:0a:95:be:8e:2f:05:3f:6c:e7:de:d1:7c:b8:a3:71:7c: - 6f:8a:05:c3:69:eb:6f:e6:76:8c:11:e1:59:0b:12:53:07:42: - 84:e8:89:ee:ab:7d:28:81:48:e8:79:d5:cf:a2:05:a4:fd:72: - 2c:7d:b4:1c:08:90:4e:0d:10:05:d1:9a:c0:69:4c:0a:14:39: - 17:fb:4d:5b:f6:42:bb:46:27:23:0f:5e:57:5b:b8:ae:9b:a3: - 0e:23:59:41:63:41:a4:f1:69:df:b3:a3:5c:10:d5:63:30:74: - a8:3c:0c:8e:1c:6b:10:e1:13:27:02:26:9b:fd:88:93:7e:91: - 9c:f9:c2:07:27:a4 + 40:d1:6d:8e:a2:0b:91:4b:a8:c4:08:d0:f3:f9:8b:d0:a3:0b: + dc:00:22:8c:f1:2e:2b:e5:e6:b4:6e:ce:9d:cf:59:32:66:6c: + bb:0e:3b:1d:9c:05:d2:eb:a6:29:f8:74:4f:dc:83:3b:32:5a: + 2c:67:86:61:25:bc:bd:19:eb:20:c6:30:69:0e:4c:b2:e3:18: + ca:9e:fe:40:bc:1c:ad:8b:03:f5:04:be:90:ce:27:27:2f:83: + 14:57:8d:4f:a0:db:46:ce:e0:7d:e2:cf:7d:ea:0c:fd:8d:00: + 27:0a:db:0d:5f:e7:1e:52:25:1f:64:b9:30:5f:07:1a:10:a3: + 69:35:0e:dc:f8:23:f7:34:07:ce:c8:92:94:39:4d:d5:c3:ab: + 33:aa:f9:67:be:66:18:ac:67:14:5f:93:5f:68:48:04:ed:1e: + c9:74:28:b2:47:34:49:11:e4:7b:38:32:e5:dc:40:13:b4:69: + 75:39:43:db:7c:4a:f0:2b:94:cd:01:ba:4d:9b:9e:68:b3:ee: + 03:9e:7f:9c:0c:cf:9c:5c:cb:d4:33:d5:f0:e3:21:54:9a:13: + 6f:eb:1a:0f:f3:8b:e8:ef:eb:34:ba:09:77:39:2a:8a:4b:e1: + 7e:9f:b5:05:be:95:b6:92:5d:4c:35:47:38:64:38:5e:27:b8: + f9:34:94:2f:57:16:b0:f5:6a:21:3f:09:34:b9:dd:f8:d1:47: + 2c:c7:5e:7f:63:49:f4:5b:f4:d9:ea:66:fc:aa:64:27:f0:72: + d7:94:6f:86:0f:e7:3b:b3:d4:d9:30:67:b8:a2:c3:f7:4d:07: + 44:b3:70:67:dd:b1:21:ac:7c:2a:04:7b:2c:1d:df:0b:82:a9: + fb:df:88:72:47:1c:f5:5d:a3:f7:52:22:2d:ea:f4:2a:45:4f: + 9b:9d:63:95:59:f3:79:05:2b:f1:5b:3b:62:71:69:90:30:d7: + 7a:b2:c8:ec:68:e5:94:bb:97:00:d0:95:a7:fd:04:6c:f7:8b: + 28:c1:96:9b:6a:94 -----BEGIN CERTIFICATE----- -MIIGvTCCBSWgAwIBAgIJAMstgJlaaVJgMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +MIIGvzCCBSegAwIBAgIJAMstgJlaaVJgMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW -MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODA4MjkxNDIzMTZaFw0zNzEwMjgx -NDIzMTZaMF0xCzAJBgNVBAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEj -MCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xEDAOBgNVBAMMB2lk -bnNhbnMwggGiMA0GCSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQCp6zt40WB3K7yj -BGugnRuqI3ApftThZWDIpvW0cVmN0nqQxsO6CCnS4dS7SYhGFiIqWjNVc2WG0gv7 -nC5DFguqbndNZk9/SjX8EOxKz4ANjd61WnTkDO5Tbiiyd+TuEBxhmbEF69bF9dtd -1Sgo8jmM7j+aa6ClYh/49bx+blJDF76EGSrmB1q+obMeZURhPXNBeoiqKR83x5Hc -LTJYMocvb6m8uABwuSka13Gb3QGu06p5ldK6TDK38HsoOy6MFO5F1PrkakG/eBHO -jcBOGPfNmTwWOqvwlcQWykr4QspWS+yTzdkgZ+mxar/yQuq7wuYSNaEfGH5yoYtV -WIgKwwZRDPqpSQuVe+J+MWLPQ6RTM+rXIHVzHtPk1f8DrgN+hSepJy/sVBBEQCzj -nyB+scn76ETWch3iyVoMj3oVOGs0b4XTDMmUw/DmEt5TDah7TqE3G+fpBIbgMSjx -MzUQZl27izmM9nQCJRAosNoNwXqlM754K9WcY6gT8kkcj1CfTmMCAwEAAaOCAo4w -ggKKMIHhBgNVHREEgdkwgdaCB2lkbnNhbnOCH3huLS1rbmlnLTVxYS5pZG4ucHl0 -aG9udGVzdC5uZXSCLnhuLS1rbmlnc2dzc2NoZW4tbGNiMHcuaWRuYTIwMDMucHl0 -aG9udGVzdC5uZXSCLnhuLS1rbmlnc2djaGVuLWI0YTNkdW4uaWRuYTIwMDgucHl0 -aG9udGVzdC5uZXSCJHhuLS1ueGFzbXE2Yi5pZG5hMjAwMy5weXRob250ZXN0Lm5l -dIIkeG4tLW54YXNtbTFjLmlkbmEyMDA4LnB5dGhvbnRlc3QubmV0MA4GA1UdDwEB -/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/ -BAIwADAdBgNVHQ4EFgQUW5NCWLC0GMxBTBXrQjNmd0xxL0IwfQYDVR0jBHYwdIAU -8+yUjvKOMMSOaMK/jmoZwMGfdmWhUaRPME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQK -DB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNh -LXNlcnZlcoIJAMstgJlaaVJbMIGDBggrBgEFBQcBAQR3MHUwPAYIKwYBBQUHMAKG -MGh0dHA6Ly90ZXN0Y2EucHl0aG9udGVzdC5uZXQvdGVzdGNhL3B5Y2FjZXJ0LmNl -cjA1BggrBgEFBQcwAYYpaHR0cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0 -Y2Evb2NzcC8wQwYDVR0fBDwwOjA4oDagNIYyaHR0cDovL3Rlc3RjYS5weXRob250 -ZXN0Lm5ldC90ZXN0Y2EvcmV2b2NhdGlvbi5jcmwwDQYJKoZIhvcNAQELBQADggGB -AF/Ym9wiVYBH4ZsEPkZTm+WnSo/rUwE51QT2z9xIhIqpw6U1Ii+rdHfspv2x5uZ0 -gjhUCyc25uw9/pIasno1DaPlfP/lWxooSykfmRs+Eeni4NfaBk/je4ytMPQ5JOit -Kg5xdKvtYumfhX5qsLtTtNdruNpUFVyaQc9h8atn1idcDKPXQecnPliJ1h8/KlLM -EwtL5ta6oMf94xekuNrMy4hwITtw3wlAbOcCgQiAsDZ3+0TFz78ZVHzRTh+iRJ7Y -Vg6/SwvghG+89sZ/NXoXyoOzgsZO0/PYMAX9bTyKq2NVb8UYumb+4jUEK652NPBW -GOhU24OyG5MKJYGB8CXKCpW+ji8FP2zn3tF8uKNxfG+KBcNp62/mdowR4VkLElMH -QoToie6rfSiBSOh51c+iBaT9cix9tBwIkE4NEAXRmsBpTAoUORf7TVv2QrtGJyMP -XldbuK6bow4jWUFjQaTxad+zo1wQ1WMwdKg8DI4caxDhEycCJpv9iJN+kZz5wgcn -pA== +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAgFw0xODA4MjkxNDIzMTZaGA8yNTI1MTAy +ODE0MjMxNlowXTELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4 +MSMwIQYDVQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEQMA4GA1UEAwwH +aWRuc2FuczCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAN4YbKyo2qzP +rj7Gw34JRkN8KA8+HNQrSUNSWQSSnRl4V5Cbf6mjZjzaC8khVqU7kKGJeM/FKAnC +i2hId7hBnx9YCWRfV3X0DzGON63HUrd2pmy2p7eY8/sdC98mVGSYC1WEVpSPX5JT +H0qzgcHguTaGopSirdOVEyl5J8lNdodOFU/spk9YHN8zul2kZifOcGnGB/KY2ulN +xc0KN5HqVSWP9MTEeJguPgTRo4NH9JTPS0E10JBFxEPD47JER7pGTfYquXfg+vV0 +0p4RHS26WEzbBEWyrDRry82zXcjfQEfhpcCvlsaAsIXa1ds/cqh6hH8aRBJZXXgq +uFPGEoPdAQnyAeOgdgqvgh1wVhxNqRsj0TdMexCNBNQwn9fKDNrs46vqk08pjXS1 +h+ntaHLJjtwH8L+sedRjR/nwhmVF0xiBwsJSXvDNTFJUQzKv0CvLxdlmXG47rb2T +a50AgbAiLZNWr5QrSQ09nIaJwYcx4SGg1zdgMwH/Nh6oyvgqvIj6aQIDAQABo4IC +jjCCAoowgeEGA1UdEQSB2TCB1oIHaWRuc2Fuc4IfeG4tLWtuaWctNXFhLmlkbi5w +eXRob250ZXN0Lm5ldIIueG4tLWtuaWdzZ3NzY2hlbi1sY2Iwdy5pZG5hMjAwMy5w +eXRob250ZXN0Lm5ldIIueG4tLWtuaWdzZ2NoZW4tYjRhM2R1bi5pZG5hMjAwOC5w +eXRob250ZXN0Lm5ldIIkeG4tLW54YXNtcTZiLmlkbmEyMDAzLnB5dGhvbnRlc3Qu +bmV0giR4bi0tbnhhc21tMWMuaWRuYTIwMDgucHl0aG9udGVzdC5uZXQwDgYDVR0P +AQH/BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMB +Af8EAjAAMB0GA1UdDgQWBBTIBpm36I/sTz1ciWoG9XcusOBqnjB9BgNVHSMEdjB0 +gBTACisoQ95fyX1H5UebNvJljGc74qFRpE8wTTELMAkGA1UEBhMCWFkxJjAkBgNV +BAoMHVB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uIENBMRYwFAYDVQQDDA1vdXIt +Y2Etc2VydmVyggkAyy2AmVppUlswgYMGCCsGAQUFBwEBBHcwdTA8BggrBgEFBQcw +AoYwaHR0cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0Y2EvcHljYWNlcnQu +Y2VyMDUGCCsGAQUFBzABhilodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rl +c3RjYS9vY3NwLzBDBgNVHR8EPDA6MDigNqA0hjJodHRwOi8vdGVzdGNhLnB5dGhv +bnRlc3QubmV0L3Rlc3RjYS9yZXZvY2F0aW9uLmNybDANBgkqhkiG9w0BAQsFAAOC +AYEAQNFtjqILkUuoxAjQ8/mL0KML3AAijPEuK+XmtG7Onc9ZMmZsuw47HZwF0uum +Kfh0T9yDOzJaLGeGYSW8vRnrIMYwaQ5MsuMYyp7+QLwcrYsD9QS+kM4nJy+DFFeN +T6DbRs7gfeLPfeoM/Y0AJwrbDV/nHlIlH2S5MF8HGhCjaTUO3Pgj9zQHzsiSlDlN +1cOrM6r5Z75mGKxnFF+TX2hIBO0eyXQoskc0SRHkezgy5dxAE7RpdTlD23xK8CuU +zQG6TZueaLPuA55/nAzPnFzL1DPV8OMhVJoTb+saD/OL6O/rNLoJdzkqikvhfp+1 +Bb6VtpJdTDVHOGQ4Xie4+TSUL1cWsPVqIT8JNLnd+NFHLMdef2NJ9Fv02epm/Kpk +J/By15Rvhg/nO7PU2TBnuKLD900HRLNwZ92xIax8KgR7LB3fC4Kp+9+Ickcc9V2j +91IiLer0KkVPm51jlVnzeQUr8Vs7YnFpkDDXerLI7GjllLuXANCVp/0EbPeLKMGW +m2qU -----END CERTIFICATE----- diff --git a/Lib/test/certdata/keycert.passwd.pem b/Lib/test/certdata/keycert.passwd.pem index 187021b8eeb..1739a3525fe 100644 --- a/Lib/test/certdata/keycert.passwd.pem +++ b/Lib/test/certdata/keycert.passwd.pem @@ -1,69 +1,69 @@ -----BEGIN ENCRYPTED PRIVATE KEY----- -MIIHbTBXBgkqhkiG9w0BBQ0wSjApBgkqhkiG9w0BBQwwHAQIc17oH9riZswCAggA -MAwGCCqGSIb3DQIJBQAwHQYJYIZIAWUDBAEqBBDwi0Mkj59S0hplpnDSNHwPBIIH -EFGdZuO4Cwzg0bspLhE1UpBN5cBq1rKbf4PyVtCczIqJt3KjO3H5I4KdQd9zihkN -A1qzMiqVZOnQZw1eWFXMdyWuCgvNe1S/PRLWY3iZfnuZ9gZXQvyMEHy4JU7pe2Ib -GNm9mzadzJtGv0YZ05Kkza20zRlOxC/cgaNUV6TPeTSwW9CR2bylxw0lTFKBph+o -uFGcAzhqQuw9vsURYJf1f1iE7bQsnWU2kKmb9cx6kaUXiGJpkUMUraBL/rShoHa0 -eet6saiFnK3XGMCIK0mhS9s92CIQV5H9oQQPo/7s6MOoUHjC/gFoWBXoIDOcN9aR -ngybosCLtofY2m14WcHXvu4NJnfnKStx73K3dy3ZLr2iyjnsqGD1OhqGEWOVG/ho -QiZEhZ+9sOnqWI2OuMhMoQJNvrLj7AY4QbdkahdjNvLjDAQSuMI2uSUDFDNfkQdy -hqF/iiEM28PmSHCapgCpzR4+VfEfXBoyBCqs973asa9qhrorfnBVxXnvsqmKNLGH -dymtEPei9scpoftE5T9TPqQj46446bXk23Xpg8QIFa8InQC2Y+yZqqlqvzCAbN6S -Qcq1DcTSAMnbmBXVu9hPmJYIYOlBMHL8JGbsGrkVOhLiiIou4w3G+DyAvIwPj6j9 -BHLqa7HgUnUEC+zL4azVHOSMqmDsOiF3w9fkBWNSkOyNoZpe+gBjbxq7sp+GjAJv -1CemRC3LSoNzLcjRG2IEGs1jlEHSSfijvwlE4lEy3JVc+QK8BOkKXXDVhY1SQHcS -pniEnj95RFVmAujdFDBoUgySyxK/y6Ju/tHPpSTG9VMNbJTKTdBWAVWWHVJzBFhR -0Ug62VrBK7fmfUdH1b37aIxqsPND2De6WLm0RX+7r3XPDJ7hm+baKCchI5CvnG19 -ky8InhMbU4qV+9LceMETmNKKDkhKl4Zx/Y3nab7DG9s/RZfrTdCHojc9Va/t0Ykp -qlVrvdj/893CdI78SW3VjWBJGWfKMyT16hBMY3TPz6ulbFXk6Pul/KcLLWslghS+ -GKZjyBe96UwfH4C7WjuIB+zo+De3Wr8xOCdJR5zwEutBMM+L/Wul8B6wIEGS71kB -TN/CAoeIgHLQFbcw4YE80dllTnSEsqF+ahVTTcCt3iLUaOgeTUxteMbXY9+nekSX -x8aUcvkMhbU9omdEowFr5/HIMKXo4UXat4fIGgh2pG8v8fA46hZXkhWUh/PhbnQw -StXzn4fA13erqVI679kHMmOIQebv4oqdcwkImrH5fEsACNjQbkYZF5fD4z+1GHkA -e2eGqejVT+OV14I8qfx9oqs2f8aqijH8fYLU0TymE7p53DYZy4WvDwk22I4rMzoQ -sGkOZwfKUYpdBI2t6tEf1ROBjoNG0E2Onq+5iooibN08rKXKAQMWsK+2vNHNHwBW -49vRheQNnRqSuLY+b7QAjA0KuRWo9YptCbnXyF/Aw64jMfAGjggDLoaZfALGZk3n -P+ZoL9xc7rYRpIca44BeYI6AhHFcWWIOX7Sm69FvmyHlfsgTAXVgY1lQPuGy68Au -PHSkgUyydDtkrfb2W2gJuqD/+h+9X2z+o/+nETYPCZm3sH5xvTY/DTcTx9kTpXxx -YQBaFTt12eVX7wZVr5K3u9M371rg+SeXC2SzL4T6APHD52cxbA1jgM0JFh3KJTuk -fADxIzM1NdzYQ45J6i2w+/Fh4VPnXZ0oiUSwE094XTBlvhI6zHgar2Q0Qx1P51vB -odd9XzyDLULuIzei0DYjTIg0KhE+wAGq1I5qtiMhmy5TdCKKNA9WGb1Pq38zpyjU -wGmztzSzCEjfLyhChaUObVRRxEfD5ioxKer/fczOhKQe8FXmGy5u/04tVmmEyNOO -JkkDtZy+UbKuJ257QnY72wPjgtHNy+S4Iv7zHUbNJNhxk+xBlRcmRNWCEM20LBSO -Tj4S9gyan+gH2+WFxy8FaENUhM+vHFEeJcjQIBFBeWICmSmdkh/r0YK1UVJ9NLfR -l0HiKm3lKg+kNCexTAPLMt2rGZ4PAKVnhVaxtuHMYYDpl2GYmyH73B9BfcPdA/Zx -GUBmd9hwcLz9SuUg+fjHcogZRRRlcZlKhw3zUCsqHSCQXZCQm7mBlG/5C/7cM7wQ -IRtsNospLStOg51gv21ClQ+uWx30XEcwmnIfVoLl1vMaguuf1u5u3dWBD/UgmqiP -1Ym8jv0BF/AS+u/CtUpwe7ZWxFT0vbyi10xxIF7O07fwFa+5dME3ycZwcyiE95K1 -ftcHlGOIhuVBMSNZXC4I9LM+7IWy+hanUcK+v5RvwBDSJV3fnAOdfrka1L/HyEEb -x/FYKEiU/TAjXDw2NtZ2itpADTSG5KbdJSwPr01Ak7aE+QYe7TIKJhBDZXGQlqq8 -1wv77zyv7V5Xq2cxSEKgSqzB9fhYZCASe8+HWlV2T+Sd +MIIHdTBfBgkqhkiG9w0BBQ0wUjAxBgkqhkiG9w0BBQwwJAQQEKGM7Z40p3TpyCvl +LTPLsQICCAAwDAYIKoZIhvcNAgkFADAdBglghkgBZQMEASoEEJAwQy7HESbbXMoF +KXadaz8EggcQBlxqFXmWQ3YlUuWTTc+dUcmikIUmI8gUSB2doBxQWLkTzaNuQ/hj +uV/eU7VkNFswxrU1j+EEkoCAtClWrQ2aKTyQSM4YfhgN1vQJdOD8DypHtd0TaRS3 +VCSkJODqcWaCEc/Ypgb1KnB6UWhit1waPq4NVCZdiSF/ueRK9/iMlIRaejg61Qn+ +LVLcegWw47zTtCZKuZqFAO3PcvfC0cresGc6hrQrb9tjUyI9qCR6MeZSrHUJQ/Pb +T9e7OREQxCLxWEUb0/tMJH5k9HoG7waU5lELOyjGKXsflEIs+uBaWIssWp69KGLi +yas7BHZc8kxJNtigFQlmECZjoKvRxJWN4fOknxI8T+ttBovOtsIlVvocBfmH+Z2m +7a5VMBxD8Phk6zU4Pk0L2S6EIUCIYGS+kgHibgxZ58/QcoDZBrKOaJ/Md+Lhk2Rn +2JCuPe8CBTkB3V+xAJdaz7JAcgxz4TqwXFXA249lfYb8qY9XXcr67O83Cxwb95lu +skmxnhAKOwKIrhS1nKiefuwN5qOiA+nuPbJFadCOO5cdOSriG3s80ugDDvlPN8k6 +4b7XLulB38R2PdH/OHuF05QGQ3kOUbJWatMig+/09LNo9FiMaCczdIXQ6LGxcHc1 +G1Y3BqK17z3yULqHvxU4tIbj5Elt+X1mfKIfifmXBUujbz5rR1pBTfpnk+DFGcHZ +B/cXWFP0tDJAE7mvna3HimNDQMP58NbhZwFtFdU3H9R5n4R5MsYQ6+a+amQ0pZ1Y +XAaQ/iEg0WY8CEFO8zEvB9R05zMm6vycNtN8CbfBq9CJ0OaR1ymMW8pRVagsLDuh +D0T5ZtWZctE2+ImlwlpGDs91CX3zDxvW/3Bwf40PF4x7LJMt/tFzqQEovokvzhUw +0jX/kf42QhvydnDoxqczdVZOfbjHELA21U+JAeAAj9jhcCEYd50c2BVt4jhST6dP +pzNAqxc1RGwTU3K3nZkWKt4hMXPDGjb0pVqiOuPz/718Nsd6ck+Ko90gFX13mPLC +6FLPGjNUmE6f3TKjbRrQz3+IWMncCyo/JSWA85rldxD0XAem64h+2XppFWJsxJD5 +nH2prLckQUWBLljWNIQQyVuAWo0TVq3NCXDFGaP6GHNrZo9zZxDhxcp6v2Me3Mir +few9aQNb59Q1/A0qwiCf26Oe8JYTD14UCez41imxD+w721SO8jzLR80+ReemdwzE +B9fXh3T/lrCIDNB+F+GWA9wQUz8mzArKbhqQhl5gk39dnLahc4aN7GrkwL4hjhKW +O3X5bBdhhRlcub9SbXFC+tx/w6G/0roGMSvD7h8G/mw3QcUUwwDvOuI7qAqQ8FC+ +xcGWeHUiTGxp2t2gz8DSaeL2TbBMbHe+tRBmKlmmIrAq3AEllWzyze9ta5ZYt46P +njZzS+vbbA0mfcFAId7tQOjQ2/ygZePs5hg91revLzNMDRJ1IeBHnrVUuDA5D1HR +iFW+hhj1Lx1s0XAJBJcDos8xy/LZlRDLxJSHrMktZOELTYh7UZw4S2PNkOPms4kP +V4G6D/0wZh0t+T9zpH57ivLdyBQ2fsFwx3X3JfQkYmoFlwCz+uUb368TunQ/yZZY +fVh9Q9ndk6y6aor0wkORsBefVv3eN+2BDt6ZUpYu3Liim+W4C/SiKVfQ+5yDb/qe +KmX7kcmr/f13BWvEOUN4fzCMiw/N0aAgEejZQTwMgivvGSMcNaBiLfdvM6hmy7NE +uvgpLtK5v+YRcOLwiaVH7UL6Q61CJ+pcDNRvbJT/iVwidYUO16zTVR6NrtPhM62a +Ziq3Q77HD4D2KCRPEo3Xivc5RwITHeDgKHwpLRB00jiZ3iR0Wm5lI5VbyWLnmQ4x +CDPAZetJZGBkbhTmUVbL0m3HNLHfF9g+LWA23L9mRNCKRemL0c0UJRRJPaNDIx8p +8h27al4xI8nFav0TZLxmSw7mqnzyE7YAoe4EsmedOSXktHWN06dnczHu8AWrgNjY +Izv9XUmy9V5vsRG/lg09FyG/eZSgfs9rdmL/qeFXbbl2J33YTqrt6O3ZaEsgg42r +piZztung8Xro3VlFSAXZGrl7Z0AWwzaOvwreCxbAG1WtBmGgLrs98z627XgfBFYA +BPvJwn9f2GZzixiN6M8c2M5XueDE/Vpn4A/GKLJ8RXXxKtbGRXFfdF/M6mEgcpa1 +9pxAhzNTPaOp+SFbm/cFa43HZYZgDg1D9zth2ZII0ZITd5OpEaSNOsrOrhqXeVQU +isBpybgPqVQ50xUuRUyoHYAZClVe6+PBYVbvlXBlTWhSPY3leUm4AYt+fjZ4VFlb +p4I2KRmd68zXXwl2spkEsC96e9k28kHta55dO32gSwaXUUHRKnFmKB24jsleF6n3 +BOKxzsO9jr6vi0jUqZXvaCnmYWqS84wZH/1S1pgotaWdMQ/KT27xwHk= -----END ENCRYPTED PRIVATE KEY----- -----BEGIN CERTIFICATE----- -MIIEgzCCAuugAwIBAgIUU+FIM/dUbCklbdDwNPd2xemDAEwwDQYJKoZIhvcNAQEL +MIIEhTCCAu2gAwIBAgIUTxhwRX6zBQc3+l3imDklEbqcDpIwDQYJKoZIhvcNAQEL BQAwXzELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYD VQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjESMBAGA1UEAwwJbG9jYWxo -b3N0MB4XDTIzMTEyNTA0MjEzNloXDTQzMDEyNDA0MjEzNlowXzELMAkGA1UEBhMC -WFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRob24gU29m -dHdhcmUgRm91bmRhdGlvbjESMBAGA1UEAwwJbG9jYWxob3N0MIIBojANBgkqhkiG -9w0BAQEFAAOCAY8AMIIBigKCAYEAzXTIl1su11AGu6sDPsoxqcRGyAX0yjxIcswF -vj+eW/fBs2GcBby95VEOKpJPKRYYB7fAEAjAKK59zFdsDX/ynxPZLqyLQocBkFVq -tclhCRZu//KZND+uQuHSx3PjGkSvK/nrGjg5T0bkM4SFeb0YdLb+0aDTKGozUC82 -oBAilNcrFz1VXpEF0qUe9QeKQhyd0MaW5T1oSn+U3RAj2MXm3TGExyZeaicpIM5O -HFlnwUxsYSDZo0jUj342MbPOZh8szZDWi042jdtSA3i8uMSplEf4O8ZPmX0JCtrz -fVjRVdaKXIjrhMNWB8K44q6AeyhqJcVHtOmPYoHDm0qIjcrurt0LZaGhmCuKimNd -njcPxW0VQmDIS/mO5+s24SK+Mpznm5q/clXEwyD8FbrtrzV5cHCE8eNkxjuQjkmi -wW9uadK1s54tDwRWMl6DRWRyxoF0an885UQWmbsgEB5aRmEx2L0JeD0/q6Iw1Nta -As8DG4AaWuYMrgZXz7XvyiMq3IxVAgMBAAGjNzA1MBQGA1UdEQQNMAuCCWxvY2Fs -aG9zdDAdBgNVHQ4EFgQUl2wd7iWE1JTZUVq2yFBKGm9N36owDQYJKoZIhvcNAQEL -BQADggGBAF0f5x6QXFbgdyLOyeAPD/1DDxNjM68fJSmNM/6vxHJeDFzK0Pja+iJo -xv54YiS9F2tiKPpejk4ujvLQgvrYrTQvliIE+7fUT0dV74wZKPdLphftT9uEo1dH -TeIld+549fqcfZCJfVPE2Ka4vfyMGij9hVfY5FoZL1Xpnq/ZGYyWZNAPbkG292p8 -KrfLZm/0fFYAhq8tG/6DX7+2btxeX4MP/49tzskcYWgOjlkknyhJ76aMG9BJ1D7F -/TIEh5ihNwRTmyt023RBz/xWiN4xBLyIlpQ6d5ECKmFNFr0qnEui6UovfCHUF6lZ -qcAQ5VFQQ2CayNlVmQ+UGmWIqANlacYWBt7Q6VqpGg24zTMec1/Pqd6X07ScSfrm -MAtywrWrU7p1aEkN5lBa4n/XKZHGYMjor/YcMdF5yjdSrZr274YYO1pafmTFwRwH -5o16c8WPc0aPvTFbkGIFT5ddxYstw+QwsBtLKE2lJ4Qfmxt0Ew/0L7xkbK1BaCOo -EGD2IF7VDQ== +b3N0MCAXDTI0MTAwODExNTExMloYDzI0MDgwMTI5MTE1MTEyWjBfMQswCQYDVQQG +EwJYWTEXMBUGA1UEBwwOQ2FzdGxlIEFudGhyYXgxIzAhBgNVBAoMGlB5dGhvbiBT +b2Z0d2FyZSBGb3VuZGF0aW9uMRIwEAYDVQQDDAlsb2NhbGhvc3QwggGiMA0GCSqG +SIb3DQEBAQUAA4IBjwAwggGKAoIBgQDwcIAYm12nmQTGB3caFn7alDe3LSliEfNC +2ZTR+5sh1eucQPbzgFM5SR4soKGElwI68Eg7g1kwqu3zmrI/FAiQI/RrUHyBZiEt +nFBPM44vY02XjUlJ9nBtgP7QjpRz6ZP0z1DrrojpYLVwg9cR0khTqD5cg2jvTB05 +yL9lQNk295/rMuueC9FaAQ+Y5la0ub7Lbe8gkYPotYliGx5qqCQmsXeTsxCpvQD7 +u0vBF9/nxlwywwzKGXabcN9YQhSECCC4c+eYjqoRgvAaF48xEnokxenzBIqZ82BR +kFo+zfNR+VhJRctNiZ6Ppa1Ise1H3LjZMDfY1S89QOLYsFXUp8L7ZMVE27Bej+Sq +4wEJ3soK/k1kr0YauqJ0lCEKkUPD9OeNHmczeakjj11tgtctsYbwgDSUYGA3w+DC +KD1aSfeuR3Hj89cSsVRrRrPFFih46Tr2PpTNoK6MtPH3RPJKV+Db8E1V2mXmNE0M +Lg6ramSHsD9iZXTLhG2JO+/876k3N3kCAwEAAaM3MDUwFAYDVR0RBA0wC4IJbG9j +YWxob3N0MB0GA1UdDgQWBBR459BlAel5MqCtG0DrvVtMZlQfuTANBgkqhkiG9w0B +AQsFAAOCAYEAaTFWjK/LFzOo+0TWqTTj4WC4N3I8JFrHnlqFJlpchYTW2z92SU1G +iEzFjWzuDNjp5KM9BqlmGtzXZvy6MItGkYsjPRdPVU0rbCmyTho6y77kTyiEG12V +UAJ1in3FOQfDwLPcp7wQRgCQq3iZlv7pwXp2Lm5fzu8kZPnxmTVdiKQun9Ps7uKq +BoM0fM2K14MxVO4Wc0SERnaPszE7xAhkIcs+NRT/gHYdTBlPhao83S3LOOdtCqCP +pNUOEaShlwI5bVsDPUXNX/eS0MYFNlsYTb5rCxK8jf3W3KNjKTOzN94pHiQOhpkg +xMPPi3m03/9oKTVXBtHI2A+u3ukheKE6sBXCLdv/GEs9zYI49zmpQxNWz5EOumWL +k+W/vPv7cD6LeHxxp+nCbEJi1gZtYb3gMY1sLkMNxcOu0QHTqHfme+k7VKWm8anO +3ogGdGtPuPAD/qjMwg3ChSDLl5Ur/E9UPlD4yM/7KtUD7mLv+jbddA62EiA9MxVB +t+yt7pOwOA66 -----END CERTIFICATE----- diff --git a/Lib/test/certdata/keycert.pem b/Lib/test/certdata/keycert.pem index a30d15ca4d6..43ad52c2af5 100644 --- a/Lib/test/certdata/keycert.pem +++ b/Lib/test/certdata/keycert.pem @@ -1,67 +1,67 @@ -----BEGIN PRIVATE KEY----- -MIIG/QIBADANBgkqhkiG9w0BAQEFAASCBucwggbjAgEAAoIBgQDNdMiXWy7XUAa7 -qwM+yjGpxEbIBfTKPEhyzAW+P55b98GzYZwFvL3lUQ4qkk8pFhgHt8AQCMAorn3M -V2wNf/KfE9kurItChwGQVWq1yWEJFm7/8pk0P65C4dLHc+MaRK8r+esaODlPRuQz -hIV5vRh0tv7RoNMoajNQLzagECKU1ysXPVVekQXSpR71B4pCHJ3QxpblPWhKf5Td -ECPYxebdMYTHJl5qJykgzk4cWWfBTGxhINmjSNSPfjYxs85mHyzNkNaLTjaN21ID -eLy4xKmUR/g7xk+ZfQkK2vN9WNFV1opciOuEw1YHwrjiroB7KGolxUe06Y9igcOb -SoiNyu6u3QtloaGYK4qKY12eNw/FbRVCYMhL+Y7n6zbhIr4ynOebmr9yVcTDIPwV -uu2vNXlwcITx42TGO5COSaLBb25p0rWzni0PBFYyXoNFZHLGgXRqfzzlRBaZuyAQ -HlpGYTHYvQl4PT+rojDU21oCzwMbgBpa5gyuBlfPte/KIyrcjFUCAwEAAQKCAYAO -M1r0+TCy4Z1hhceu5JdLql0RELZTbxi71IW2GVwW87gv75hy3hGLAs/1mdC+YIBP -MkBka1JqzWq0/7rgcP5CSAMsInFqqv2s7fZ286ERGXuZFbnInnkrNsQUlJo3E9W+ -tqKtGIM/i0EVHX0DRdJlqMtSjmjh43tB+M1wAUV+n6OjEtJue5wZK+AIpBmGicdP -qZY+6IBnm8tcfzPXFRCoq7ZHdIu0jxnc4l2MQJK3DdL04KoiStOkSl8xDsI+lTtq -D3qa41LE0TY8X2jJ/w6KK3cUeK7F4DQYs+kfCKWMVPpn0/5u6TbC1F7gLvkrseph -7cIgrruNNs9iKacnR1w3U72R+hNxHsNfo4RGHFa192p/Mfc+kiBd5RNR/M9oHdeq -U6T/+KM+QyF5dDOyonY0QjwfAcEx+ZsV72nj8AerjM907I6dgHo/9YZ2S1Dt/xuG -ntD+76GDzmrOvXmmpF0DsTn+Wql7AC4uzaOjv6PVziqz03pR61RpjPDemyJEWMkC -gcEA7BkGGX3enBENs3X6BYFoeXfGO/hV7/aNpA6ykLzw657dqwy2b6bWLiIaqZdZ -u0oiY6+SpOtavkZBFTq4bTVD58FHL0n73Yvvaft507kijpYBrxyDOfTJOETv+dVG -XiY8AUSAE6GjPi0ebuYIVUxoDnMeWDuRJNvTck4byn1hJ1aVlEhwXNxt/nAjq48s -5QDuR6Z9F8lqEACRYCHSMQYFm35c7c1pPsHJnElX8a7eZ9lT7HGPXHaf/ypMkOzo -dvJNAoHBAN7GhDomff/kSgQLyzmqKqQowTZlyihnReapygwr8YpNcqKDqq6VlnfH -Jl1+qtSMSVI0csmccwJWkz1WtSjDsvY+oMdv4gUK3028vQAMQZo+Sh7OElFPFET3 -UmL+Nh73ACPgpiommsdLZQPcIqpWNT5NzO+Jm5xa+U9ToVZgQ7xjrqee5NUiMutr -r7UWAz7vDWu3x7bzYRRdUJxU18NogGbFGWJ1KM0c67GUXu2E7wBQdjVdS78UWs+4 -XBxKQkG2KQKBwQCtO+M82x122BB8iGkulvhogBjlMd8klnzxTpN5HhmMWWH+uvI1 -1G29Jer4WwRNJyU6jb4E4mgPyw7AG/jssLOlniy0Jw32TlIaKpoGXwZbJvgPW9Vx -tgnbDsIiR3o9ZMKMj42GWgike4ikCIc+xzRmvdMbHIHwUJfCfEtp9TtPGPnh9pDz -og3XLsMNg52GXnt3+VI6HOCE41XH+qj2rZt5r2tSVXEOyjQ7R5mOzSeFfXJVwDFX -v/a/zHKnuB0OAdUCgcBLrxPTEaqy2eMPdtZHM/mipbnmejRw/4zu7XYYJoG7483z -SlodT/K7pKvzDYqKBVMPm4P33K/x9mm1aBTJ0ZqmL+a9etRFtEjjByEKuB89gLX7 -uzTb7MrNF10lBopqgK3KgpLRNSZWWNXrtskMJ5eVICdkpdJ5Dyst+RKR3siEYzU9 -+yxxAFpeQsqB8gWORva/RsOR8yNjIMS3J9fZqlIdGA8ktPr0nEOyo96QQR5VdACE -5rpKI2cqtM6OSegynOkCgcAnr2Xzjef6tdcrxrQrq0DjEFTMoCAxQRa6tuF/NYHV -AK70Y4hBNX84Bvym4hmfbMUEuOCJU+QHQf/iDQrHXPhtX3X2/t8M+AlIzmwLKf2o -VwCYnZ8SqiwSaWVg+GANWLh0JuKn/ZYyR8urR79dAXFfp0UK+N39vIxNoBisBf+F -G8mca7zx3UtK2eOW8WgGHz+Y20VZy0m/nkNekd1ZTXoSGhL+iN4XsTRn1YQIn69R -kNdcwhtZZ3dpChUdf+w/LIc= +MIIG/gIBADANBgkqhkiG9w0BAQEFAASCBugwggbkAgEAAoIBgQDwcIAYm12nmQTG +B3caFn7alDe3LSliEfNC2ZTR+5sh1eucQPbzgFM5SR4soKGElwI68Eg7g1kwqu3z +mrI/FAiQI/RrUHyBZiEtnFBPM44vY02XjUlJ9nBtgP7QjpRz6ZP0z1DrrojpYLVw +g9cR0khTqD5cg2jvTB05yL9lQNk295/rMuueC9FaAQ+Y5la0ub7Lbe8gkYPotYli +Gx5qqCQmsXeTsxCpvQD7u0vBF9/nxlwywwzKGXabcN9YQhSECCC4c+eYjqoRgvAa +F48xEnokxenzBIqZ82BRkFo+zfNR+VhJRctNiZ6Ppa1Ise1H3LjZMDfY1S89QOLY +sFXUp8L7ZMVE27Bej+Sq4wEJ3soK/k1kr0YauqJ0lCEKkUPD9OeNHmczeakjj11t +gtctsYbwgDSUYGA3w+DCKD1aSfeuR3Hj89cSsVRrRrPFFih46Tr2PpTNoK6MtPH3 +RPJKV+Db8E1V2mXmNE0MLg6ramSHsD9iZXTLhG2JO+/876k3N3kCAwEAAQKCAYAK +Ap0KqTlCd4fv2LK4NtSMNByHt00gRKAMmfNstJ12UKoxBLFjXOXaHjWv5PYkh4bz +vjo7pBHMCWnDuR6Pqr1ahuyvpRex6XcbJ4VebsaOKYO6+gphlm2C2ZqCQ1Vh6Akd +aZ40Wb1gfgK/zvVezBLvzLLf9iahw9j5pWZ2iDci5zdUuvd9Sn+qUB3+nyRf/NW5 +MXgBsp07zIcOOxPOm/Z5V+0jDJL2hiRq1pbmUKClTShcgqtfJKU//niF+6ZQAuiJ +LBPaKIdPXyxLYnkyq2IgjPU0ZwxzdP0a2k72kvImd25Daj7elhGr3++IR+nFzt6h +vqflOfmKDF3zZPyUVI3YXjxo/FrOwGbLMHuuHBgE9txH/mOl1gByrxP+Ax18i3Bf +spSLeUvtaf/w/MopyspPoJBRbAM06PUHQI2v9xq3BZL/gHe2CdJPds2WzpaaVFG4 +oJWNrE3s6CowLqUkqzB7LqJ4ReZ6xe6SpkRotdmVknlIKgDenTFeEUEEVyBiFQEC +gcEA/F1GAaBG0e9vB+AOHZ96SLlZVzObSBYq2kVwUhhGItNnyU9c3fWPIrGREKQa +lw5dsvjl58ij5uEtJoPZf5BsJ0q6xHAs/kKxfpNfZAeoKAV96Z6MVcY+6WOyGjPF +aQo+GgSrCPIciW//WXZrWI1t0M2G0vZ5CFNohnKod+cSgV03PAActlyM2H+r7dtm +MpAD3EPWeeA75saKj/x0SOzuL/wzXKR8BZ6CINZ6r61Tcbk2mDwOHPhUrHeCwjoU +nhy5AoHBAPPnP2FSXFCPXD1Z1hFInCFgm41j7LEyBTVLlnqUrRk7i18fi/WcwwLH ++XvM5DcONY/V3sh7a3tZeHN1P70tRxLE0oO51D4tP5im/oZ6L+hszSYXX7lCbJSR +tni6nU1dssW3nmswfUn01Oh+B0rBGon3RQB6x4beTAW0piVxg9Ic2HYucS1Scrqw +afiFQ5KWklnMYJKInPFzlCwMdgBCuue1dZoJstU9nLQALNNSpGXB2X0+7j9D/qkz +Caw5MfgQwQKBwQDzdCvP78XCSuBq0XvsmefG9n+4fwGDFld6v9gualpmyFjsPJKT +UYwm5PPUAOvh46sCt9hatRVg6sO6zyFoTXP4p7/rN2hAVSiTuiog/r369elVEW3C +ZYBVeKbdXipIPehRA0XYWHCtKY1Fydae07kn4M37AGkcXhKM+VmKajFQ+RMK3/TS +/A+n3+qFiM1bY9FFkW/7nRVMeSY850dq/p59TihibA91AEf6084BYg0IvatsSys2 +SV6uDpDnPE6dhYkCgcBECtAwq1RbmRLnfqdsnPAJk7Txhd3jNQwk6RhqzA1aS7U+ +7UMTWw9AOF+OPQOxpEInBUgob931RGmI9D263eXFA6mi2/Ws/tyODpBVHcM9uRSm +OsEWosQ90kSwe4ckrS4RYH9OcfGR7z5yOa55GVP5B0V1s8r0AhH9SX9MVNWsiSWO +GriyJx0gndSCY1MNkvnzGUQbvQbjiRXeD//fZL5Vo9bSCUCdopmT0bSvo49/X8v3 +19WJSsPBmh5psG8TQEECgcEA64CqZpPux35LeLQsKe0fYeNfAncqiaIoRbAvxKCi +SQf27SD8HK+sfvhvYY7bP7TMEeM7B/O2/AqBQQP0UARIGJg2AknBQT0A7//yJu+o +v4FHy2XKh+RMAx7QrdvnQ4CfrjvjQIaAcN1HrdTKWwgQZZImRf57nUCMm82ktZ2k +vYEJTXMkT8CY0DSeGtPmX5ynk7cauHTdZrkPGhZ3Hr6GAFomOammnnytv2wc+5FA +Ap+d65UgF4KjGY4rtsS+jOHn -----END PRIVATE KEY----- -----BEGIN CERTIFICATE----- -MIIEgzCCAuugAwIBAgIUU+FIM/dUbCklbdDwNPd2xemDAEwwDQYJKoZIhvcNAQEL +MIIEhTCCAu2gAwIBAgIUTxhwRX6zBQc3+l3imDklEbqcDpIwDQYJKoZIhvcNAQEL BQAwXzELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYD VQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjESMBAGA1UEAwwJbG9jYWxo -b3N0MB4XDTIzMTEyNTA0MjEzNloXDTQzMDEyNDA0MjEzNlowXzELMAkGA1UEBhMC -WFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRob24gU29m -dHdhcmUgRm91bmRhdGlvbjESMBAGA1UEAwwJbG9jYWxob3N0MIIBojANBgkqhkiG -9w0BAQEFAAOCAY8AMIIBigKCAYEAzXTIl1su11AGu6sDPsoxqcRGyAX0yjxIcswF -vj+eW/fBs2GcBby95VEOKpJPKRYYB7fAEAjAKK59zFdsDX/ynxPZLqyLQocBkFVq -tclhCRZu//KZND+uQuHSx3PjGkSvK/nrGjg5T0bkM4SFeb0YdLb+0aDTKGozUC82 -oBAilNcrFz1VXpEF0qUe9QeKQhyd0MaW5T1oSn+U3RAj2MXm3TGExyZeaicpIM5O -HFlnwUxsYSDZo0jUj342MbPOZh8szZDWi042jdtSA3i8uMSplEf4O8ZPmX0JCtrz -fVjRVdaKXIjrhMNWB8K44q6AeyhqJcVHtOmPYoHDm0qIjcrurt0LZaGhmCuKimNd -njcPxW0VQmDIS/mO5+s24SK+Mpznm5q/clXEwyD8FbrtrzV5cHCE8eNkxjuQjkmi -wW9uadK1s54tDwRWMl6DRWRyxoF0an885UQWmbsgEB5aRmEx2L0JeD0/q6Iw1Nta -As8DG4AaWuYMrgZXz7XvyiMq3IxVAgMBAAGjNzA1MBQGA1UdEQQNMAuCCWxvY2Fs -aG9zdDAdBgNVHQ4EFgQUl2wd7iWE1JTZUVq2yFBKGm9N36owDQYJKoZIhvcNAQEL -BQADggGBAF0f5x6QXFbgdyLOyeAPD/1DDxNjM68fJSmNM/6vxHJeDFzK0Pja+iJo -xv54YiS9F2tiKPpejk4ujvLQgvrYrTQvliIE+7fUT0dV74wZKPdLphftT9uEo1dH -TeIld+549fqcfZCJfVPE2Ka4vfyMGij9hVfY5FoZL1Xpnq/ZGYyWZNAPbkG292p8 -KrfLZm/0fFYAhq8tG/6DX7+2btxeX4MP/49tzskcYWgOjlkknyhJ76aMG9BJ1D7F -/TIEh5ihNwRTmyt023RBz/xWiN4xBLyIlpQ6d5ECKmFNFr0qnEui6UovfCHUF6lZ -qcAQ5VFQQ2CayNlVmQ+UGmWIqANlacYWBt7Q6VqpGg24zTMec1/Pqd6X07ScSfrm -MAtywrWrU7p1aEkN5lBa4n/XKZHGYMjor/YcMdF5yjdSrZr274YYO1pafmTFwRwH -5o16c8WPc0aPvTFbkGIFT5ddxYstw+QwsBtLKE2lJ4Qfmxt0Ew/0L7xkbK1BaCOo -EGD2IF7VDQ== +b3N0MCAXDTI0MTAwODExNTExMloYDzI0MDgwMTI5MTE1MTEyWjBfMQswCQYDVQQG +EwJYWTEXMBUGA1UEBwwOQ2FzdGxlIEFudGhyYXgxIzAhBgNVBAoMGlB5dGhvbiBT +b2Z0d2FyZSBGb3VuZGF0aW9uMRIwEAYDVQQDDAlsb2NhbGhvc3QwggGiMA0GCSqG +SIb3DQEBAQUAA4IBjwAwggGKAoIBgQDwcIAYm12nmQTGB3caFn7alDe3LSliEfNC +2ZTR+5sh1eucQPbzgFM5SR4soKGElwI68Eg7g1kwqu3zmrI/FAiQI/RrUHyBZiEt +nFBPM44vY02XjUlJ9nBtgP7QjpRz6ZP0z1DrrojpYLVwg9cR0khTqD5cg2jvTB05 +yL9lQNk295/rMuueC9FaAQ+Y5la0ub7Lbe8gkYPotYliGx5qqCQmsXeTsxCpvQD7 +u0vBF9/nxlwywwzKGXabcN9YQhSECCC4c+eYjqoRgvAaF48xEnokxenzBIqZ82BR +kFo+zfNR+VhJRctNiZ6Ppa1Ise1H3LjZMDfY1S89QOLYsFXUp8L7ZMVE27Bej+Sq +4wEJ3soK/k1kr0YauqJ0lCEKkUPD9OeNHmczeakjj11tgtctsYbwgDSUYGA3w+DC +KD1aSfeuR3Hj89cSsVRrRrPFFih46Tr2PpTNoK6MtPH3RPJKV+Db8E1V2mXmNE0M +Lg6ramSHsD9iZXTLhG2JO+/876k3N3kCAwEAAaM3MDUwFAYDVR0RBA0wC4IJbG9j +YWxob3N0MB0GA1UdDgQWBBR459BlAel5MqCtG0DrvVtMZlQfuTANBgkqhkiG9w0B +AQsFAAOCAYEAaTFWjK/LFzOo+0TWqTTj4WC4N3I8JFrHnlqFJlpchYTW2z92SU1G +iEzFjWzuDNjp5KM9BqlmGtzXZvy6MItGkYsjPRdPVU0rbCmyTho6y77kTyiEG12V +UAJ1in3FOQfDwLPcp7wQRgCQq3iZlv7pwXp2Lm5fzu8kZPnxmTVdiKQun9Ps7uKq +BoM0fM2K14MxVO4Wc0SERnaPszE7xAhkIcs+NRT/gHYdTBlPhao83S3LOOdtCqCP +pNUOEaShlwI5bVsDPUXNX/eS0MYFNlsYTb5rCxK8jf3W3KNjKTOzN94pHiQOhpkg +xMPPi3m03/9oKTVXBtHI2A+u3ukheKE6sBXCLdv/GEs9zYI49zmpQxNWz5EOumWL +k+W/vPv7cD6LeHxxp+nCbEJi1gZtYb3gMY1sLkMNxcOu0QHTqHfme+k7VKWm8anO +3ogGdGtPuPAD/qjMwg3ChSDLl5Ur/E9UPlD4yM/7KtUD7mLv+jbddA62EiA9MxVB +t+yt7pOwOA66 -----END CERTIFICATE----- diff --git a/Lib/test/certdata/keycert.pem.reference b/Lib/test/certdata/keycert.pem.reference new file mode 100644 index 00000000000..5528b98fff9 --- /dev/null +++ b/Lib/test/certdata/keycert.pem.reference @@ -0,0 +1,13 @@ +{'issuer': ((('countryName', 'XY'),), + (('localityName', 'Castle Anthrax'),), + (('organizationName', 'Python Software Foundation'),), + (('commonName', 'localhost'),)), + 'notAfter': 'Jan 29 11:51:12 2408 GMT', + 'notBefore': 'Oct 8 11:51:12 2024 GMT', + 'serialNumber': '4F1870457EB3050737FA5DE298392511BA9C0E92', + 'subject': ((('countryName', 'XY'),), + (('localityName', 'Castle Anthrax'),), + (('organizationName', 'Python Software Foundation'),), + (('commonName', 'localhost'),)), + 'subjectAltName': (('DNS', 'localhost'),), + 'version': 3} diff --git a/Lib/test/certdata/keycert2.pem b/Lib/test/certdata/keycert2.pem index c7c4aa74583..63e2181ed83 100644 --- a/Lib/test/certdata/keycert2.pem +++ b/Lib/test/certdata/keycert2.pem @@ -1,67 +1,67 @@ -----BEGIN PRIVATE KEY----- -MIIG/gIBADANBgkqhkiG9w0BAQEFAASCBugwggbkAgEAAoIBgQCyAUXjczgUEn7m -mOwDMi/++wDRxqJAJ2f7F9ADxTuOm+EtdpfYr4mBn8Uz9e3I+ZheG5y3QZ1ddBYA -9YTfcUL0on8UXLOOBVZCetxsQXoSAuDMPV0IXeEgtZZDXe7STqKSQeYk7Cz+VtHe -lZ8j7oOOcx5sJgpbaD+OGJnPoAdB8l8nQfxqAG45sW4P6gfLKoJLviKctDe5pvgi -JC8tvytg/IhESKeefLZ4ix2dNjj2GNUaL+khU6UEuM1kJHcPVjPoYc+y8fop/qhQ -0ithBhO2OvJ+YmOFdCE67SyCwU3p8zJpN+XkwbHttgmNg4OSs7H6V7E52/CsTNTA -SthBHXtxqaM+vjbGARrz2Fpc/n+LwRt7MGIR0gVtntTgnP0HoeHskhAIeDtaPrZ6 -zHdl3aDwgAecVebTEBT5YPboz+X1lWdOrRD2JW3bqXSRIN3E4qz5IMuNx3VvhpSR -eFZzR6QIbxQqzO/Vp93Ivy8hPZ6WMgfSYWs7CGtu4NP79PJfdMsCAwEAAQKCAYAc -e3yp2NlbyNvaXRTCrCim5ZXrexuiJUwLjvNfbxNJDeM5iZThfLEFd0GwP0U1l86M -HGH2pr6d4gHVVHPW5wIeL9Qit3SZoHv9djhH8DAuqpw6wgTdXlw0BipNjD23FBMK -URYYyVuntM+vDITi1Hrjc8Ml7e5RUvx8aa5O3R3cLQKRvwq7EWeRvrTMQhfOJ/ai -VQGnzmRuRevFVsHf0YuI4M+TEYcUooL2BdiOu8rggfezUYA9r2sjtshSok0UvKeb -79pNzWmg9EWVeFk+A0HQpyLq+3EVyB5UZ3CZRkT0XhEm1B7mpKrtcGMjaumNAam7 -jkhidGdhT/PV9BB1TttcqwTf+JH9P9sSpY9ZTA1LkkeWe9Rwqpxjjssqxxl6Xnds -+wUfuovVvHuBinsO+5PLE5UQByn21WcIBNnPCAMvALy/68T7z8+ATJ+I2CqBXuM2 -z5663hNrvdCu93PpK4j/k/1l3NTrthaorbnPhkmNYHQkBicpAfFQywrv6enD+30C -gcEA7Vlw76og4oxI7SSD6gTfo85OqTLp2CUZhNNxzYiUOOssRnGHBKsGZ8p0OhLN -vk9/SgbeHL5jsmnyd8ZuYWmsOQHRWgg1zO3S25iuq+VAo4cL/7IynoJ0RP5xP0Pw -QD+xJLZQp0XuLUtXnlc6dM5Hg7tOTthOP9UxA1i57lzpYfkRnKmSeWi+4IDJymOt -WoWnEK7Yr7qSg6aScLWRyIvAPVmKF9LToSFaTq0eOD0GIwAQxqNbIwN3U0UJ5Ruc -KRBVAoHBAL/+DNGqnEzhhWS6zqZp2eH90YR+b3R4yOK4PROm2AVA3h1GhIAiWX68 -PvKYZK9dZ9EdAswlFf9PVQYIXUraR3az0UiIywnNMri+kO1ZxwofGvljrOfRRLg0 -B46wuHi6dVgTWzjTl503G9+FpAYNHv184xsr1tne0pf2TKEnN7oyQciCV8qtr8vV -HrL46uaD0w1fcXIXbO3F/7ErLsvsgLzKfxR5BeQo6Fq0GmzD+lCmzVNirtfl2CZj -2ukROXUQnwKBwQDR1CqFlm/wGHk4PPnp31ke5XqhFoOpNFM1HAEV5VK0ZyQDOsZU -mCXXiCHsXUdKodk0RpIB80cMKaHTxbc7o0JAO50q7OszOmUZAggZq1jTuMYgzRb3 -DvlfLVpMxfEVu7kNbagr2STRIjRZpV/md569lM+L4Kp8wCrOfJgTZExm8txhFYCK -mNF2hCThKfHNfy7NDuY9pMF2ZcI8pig1lWbkVc5BdX7miifeOinnKfvM4XfzQ+OE -NsI8+WHgC+KoYukCgcBwrOpdCmHchOZCbZfl9m1Wwh16QrGqi1BqLnI53EsfGijA -yaftgzs+s7/FpEZC3PCWuw3vPTyhr69YcQQ/b8dNFM8YYJ+4SuMfpUds5Kl5eTPd -dO/+xMQtzus4hOJeiB9h50o8GYH7VGJZVhcjLgQoBGlMgvf+uVSitnvWgCumbORK -hqR7YF+xoov3wToquubcDE2KBdF54h/jnFJEf7I2GilmnHgmpRNoWBbCCmoXdy09 -aMbwEgY+0Y+iBOfRmkUCgcEAoHJLw7VnZQGQQL4l3lnoGU9o06RPkNbpda9G/Ptz -v+K7DXmHiLFVDszvZaPVreohuc0tKdrT0cZpZ21h0GQD4B6JX66R/y6CCAr0QpdA -pFZO9sc5ky6RJ4xZCoCsNJzORNUb36cagEzBWExb7Jz2v6gNa044K5bs3CVv5h15 -rJtTxZNn/gcnIk+gt//67WUnKLS4PR5PVCCqYhSbhFwx/OvVTJmflIBUinAclf2Q -M4mhHOfwBicqYzzEYbOE9Vk9 +MIIG/gIBADANBgkqhkiG9w0BAQEFAASCBugwggbkAgEAAoIBgQCtFmmyQ+HjjeBX +UE61fTenp2C9cvHzZSTFvKNeKZ2ZVZzH1HFnMBlTXGlXybSi2PhfxNYo5JrNGW4v +ZADh/eIFOONvPebQBONi4B1188Mb1RHgyIMtl/gOLF+L9DkStFb2eJ3fFInjNh5b +yhhpcOH1AmPCsiLeCAO3qc9qeChmJ4hAy1xmx74yQkOGzu0hKZ9d5QNNemCpPKTh +J+X8+PCY+FRObAdKJ56er2BjkdaUB/Mp6BBLokYs917VdnH6TiicqfrUsjh36XpO +G35/bW9jsQRpbeGkMIW6vn9GEYT79d4+FWdl9SxTfcSy7tn35/SvtIYDByWLc4IJ +izTuFV91pGmsXSav4YVF1hLC5cy8iEfKrdAl1kzamHfHKuY+P+sbwSQHPn5Z5xyU +Vr+odU0t7+YExBKLG8zHkQ+h2bwEeFy6H9uYQZFhBLOHrkkWSPYLKjzch6DyNkTp ++5cGWLOkcs8Xt0mjG2gzUlgwIYWHkq7wE9jxz3/tJ3MGf2CNDxsCAwEAAQKCAYAU +x+HCxFb6LLjS4tJrBHCf0VRSX9rf/7Ri2oYRXRUN1B3cTrZFeZyLdJyKggNiSy3E +GcF4lrxl2Xgaz8/cp3rxE5EYHv93hCQPZUbu4r2kZCLJsRPcGn3D/dyEKhCGyAi/ +fgsTkxyxnyK/9HXqhYyrZNu4rrh0SH63GKJ4GZsIFhZME95b0f6/e/3YJpSJFxKK +LF+xd/rjoWNOS62s1JqA/czpaXDqnJ0frNX5aQixLHnWUy/5qgRGNnY3/LOB0MuU +EjFizRSJp6gQQ64dIFgiMyUMi+Z6fw6xkTpN5EbmgFCwmaja1/hp9UZ+atlVpJsO +SB+OhF2wxB+bX2phGHJ9DD4RY/di1YKPvmH0+LITq4JiJWwJLgoCm7krzfVxZFZN +lgZPvdNw/Yt/Y3bN31FMw8DwRCThnL+MH83E58iCWQq2zKS1RssBBwFacI6Wf1ed +442NPIH47oi5qRKUxYHze6kFssSs4TkbR++VYHXLbTLjqfePLP92ZFxdOstx5jEC +gcEA48mx39dwr1oSk4uR+q8aDZIjPT+9D0IxcdAmL968QgG98hF2Tx7s5alR4y+9 +2FueR5fSqRweJfaDn77j5DVNXLD5jBz73pdwIkfqJHLfcsN7NnzCz3mxU9WP2F7R +W8KD3C2MtzGwQwOI3GT20+wJaA7p5n5zKR7AC0RxhC3U4zwH5nSjtY70A3dWXkn3 +F0j+B6ak0P0oN9VQUuvk5zw0IZ32uxef7poiHvFZpl1DQSgX/edT1CIT1s0nmVFl +w1ojAoHBAMKGX+TobGZfZFj/I7oIVQ7023pi516fZx/R0Y0FTmZzIKsdt3wrPmar +3K5dZGasNd0wzUsxX+vSjMtI1oERuPd4Rs6YqQHpX892uJ7L41/oBcFCc+Fk7jBY +TiRg/8/gOTaQe1nDR52K/R0sqBatRqInNMBDcBsVguBC4dTqP8CQ0CnEjAOYsk1s +1nYPKcfM36BUXBmSlnvUPWMm8KCT9Kko9ylHVq+d+ziDebfYpy6fTvti2NM31wTM +1prasEUaqQKBwQDQX14+5MapMd1SaVelmW5cwbVIvzjEb4npkj6MhdVzEELg4IZ4 +hFKzGDvXdoHVHKJi3YiQuC8ADUyE4kt4JCZbx2zQdmcVTTT/tweCRi8PvbDFvEU9 +JBZKGU+X38zmgr66uFRD9MlH1EDrU9TTMdW9Af+HoV7ZW87Tv82T25UmNXEIqORl +HpsrXIx+fmzxOQ1glFmq8BpNUO5EnJPtz43kvqrIpSjhTNAvvBqFbEUsom+oDWgK +4w2A7nTt9J8BoD8CgcBkYKa3HmBhazQC4JV097u8nglrXAH8R9EVEFZLqMNOBnaD +FjCKeF4Y6PJVX4fhm1eoLfihpnbS37EbbRiTPavutzgCf7AmdmCkU6Ts/FT2NmpR +0ZKuakCm3cpk51DZ2eBsEZ41MZmQ6Bm4pkSOfxeFsSl9VM9SioUgaCLUlZQUMCXa +h7ugV3kajuETxrtOiJ+UwjNMVuIkP971fTCKDA8iAyuXN2K5+JGcFewHPFr4qeg9 +vEIarCPeLD1JZzOyVRECgcEAhqeXfmFezNKGvr7qCflJe92LF9MStgJ7yE0OfADy +B+RZKeOwoqOO3VFR1piAn/DzrC2K9q+Of61gw+KWQOgMnsigqZ2mFLGChRjWhb7S +3G0DGOb4+DD9RR6wlFPFXSwVxSWGKrqNhOJik/IzVWvYxOOFjVt1adXZ3GftnYsv +nZCsS94H4kMiXr6UkbkjjxnZ1WkE9DaQJU27Mw1dtwb1ECzO0rty9B9L6b80MYW/ +nMhJV6sanCUex4nAChHD+VR7 -----END PRIVATE KEY----- -----BEGIN CERTIFICATE----- -MIIEjDCCAvSgAwIBAgIUQ2S3jJ5nve5k5956sgsrWY3vw9MwDQYJKoZIhvcNAQEL +MIIEjjCCAvagAwIBAgIUCP/QP57z62jCsjf6l0ft/yzDxOkwDQYJKoZIhvcNAQEL BQAwYjELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYD VQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEVMBMGA1UEAwwMZmFrZWhv -c3RuYW1lMB4XDTIzMTEyNTA0MjEzN1oXDTQzMDEyNDA0MjEzN1owYjELMAkGA1UE -BhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRob24g -U29mdHdhcmUgRm91bmRhdGlvbjEVMBMGA1UEAwwMZmFrZWhvc3RuYW1lMIIBojAN -BgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEAsgFF43M4FBJ+5pjsAzIv/vsA0cai -QCdn+xfQA8U7jpvhLXaX2K+JgZ/FM/XtyPmYXhuct0GdXXQWAPWE33FC9KJ/FFyz -jgVWQnrcbEF6EgLgzD1dCF3hILWWQ13u0k6ikkHmJOws/lbR3pWfI+6DjnMebCYK -W2g/jhiZz6AHQfJfJ0H8agBuObFuD+oHyyqCS74inLQ3uab4IiQvLb8rYPyIREin -nny2eIsdnTY49hjVGi/pIVOlBLjNZCR3D1Yz6GHPsvH6Kf6oUNIrYQYTtjryfmJj -hXQhOu0sgsFN6fMyaTfl5MGx7bYJjYODkrOx+lexOdvwrEzUwErYQR17camjPr42 -xgEa89haXP5/i8EbezBiEdIFbZ7U4Jz9B6Hh7JIQCHg7Wj62esx3Zd2g8IAHnFXm -0xAU+WD26M/l9ZVnTq0Q9iVt26l0kSDdxOKs+SDLjcd1b4aUkXhWc0ekCG8UKszv -1afdyL8vIT2eljIH0mFrOwhrbuDT+/TyX3TLAgMBAAGjOjA4MBcGA1UdEQQQMA6C -DGZha2Vob3N0bmFtZTAdBgNVHQ4EFgQU5wVOIuQD/Jxmam/97g91+igosWQwDQYJ -KoZIhvcNAQELBQADggGBAFv5gW5x4ET5NXEw6vILlOtwxwplEbU/x6eUVR/AXtEz -jtq9zIk2svX/JIzSLRQnjJmb/nCDCeNcFMkkgIiB64I3yMJT9n50fO4EhSGEaITZ -vYAw0/U6QXw+B1VS1ijNA44X2zvC+aw1q9W+0SKtvnu7l16TQ654ey0Qh9hOF1HS -AZQ46593T9gaZMeexz4CShoBZ80oFOJezfNhyT3FK6tzXNbkVoJDhlLvr/ep81GG -mABUGtKQYYMhuSSp0TDvf7jnXxtQcZI5lQOxZp0fnWUcK4gMVJqFVicwY8NiOhAG -6TlvXYP4COLAvGmqBB+xUhekIS0jVzaMyek+hKK0sT/OE+W/fR5V9YT5QlHFJCf5 -IUIfDCpBZrBpsOTwsUm8eL0krLiBjYf0HgH5oFBc7aF4w1kuUJjlsJ68bzO9mLEF -HXDaOWbe00+7BMMDnyuEyLN8KaAGiN8x0NQRX+nTAjCdPs6E0NftcXtznWBID6tA -j5m7qjsoGurj6TlDsBJb1A== +c3RuYW1lMCAXDTI0MTAwODExNTExMloYDzI0MDgwMTI5MTE1MTEyWjBiMQswCQYD +VQQGEwJYWTEXMBUGA1UEBwwOQ2FzdGxlIEFudGhyYXgxIzAhBgNVBAoMGlB5dGhv +biBTb2Z0d2FyZSBGb3VuZGF0aW9uMRUwEwYDVQQDDAxmYWtlaG9zdG5hbWUwggGi +MA0GCSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQCtFmmyQ+HjjeBXUE61fTenp2C9 +cvHzZSTFvKNeKZ2ZVZzH1HFnMBlTXGlXybSi2PhfxNYo5JrNGW4vZADh/eIFOONv +PebQBONi4B1188Mb1RHgyIMtl/gOLF+L9DkStFb2eJ3fFInjNh5byhhpcOH1AmPC +siLeCAO3qc9qeChmJ4hAy1xmx74yQkOGzu0hKZ9d5QNNemCpPKThJ+X8+PCY+FRO +bAdKJ56er2BjkdaUB/Mp6BBLokYs917VdnH6TiicqfrUsjh36XpOG35/bW9jsQRp +beGkMIW6vn9GEYT79d4+FWdl9SxTfcSy7tn35/SvtIYDByWLc4IJizTuFV91pGms +XSav4YVF1hLC5cy8iEfKrdAl1kzamHfHKuY+P+sbwSQHPn5Z5xyUVr+odU0t7+YE +xBKLG8zHkQ+h2bwEeFy6H9uYQZFhBLOHrkkWSPYLKjzch6DyNkTp+5cGWLOkcs8X +t0mjG2gzUlgwIYWHkq7wE9jxz3/tJ3MGf2CNDxsCAwEAAaM6MDgwFwYDVR0RBBAw +DoIMZmFrZWhvc3RuYW1lMB0GA1UdDgQWBBSAf/Z+TUHMOcr020NGG4geSnlNhTAN +BgkqhkiG9w0BAQsFAAOCAYEAIia0ULQmC510p78HZbiOb2BV8q+0AY0l6Cn/FRIL +Voqy7uB5oEYdDisla8epmBTU35+JmfrRHbP6IuSqKdcGj8VqsjZljhntSrB7rw4O +IqjbnxnfEuREjY2w+WGLvvdtVGXCBgfRmIItORFKpoOvLzLIi1lXeDq8QL97K4nM +tPsZpzILCBCoXmhh0MweCCNe1HSD8q6EbSDjVoxraBl0YK/l1ID08Fi5fwdddRX7 +txfx5NtoWWbsLZY0GdBtQmRCifs7P9lyhjUd87bJGd8WCBE4IzLwJiHEyNPI4WHe +8jmvKD/zO7mKxT9/jF7IBqwTHgQQ1uaRGytiXjRlfPu4Ez8ASR6rP7Hlua7h0Ba7 +OcDPcgM8rzN4zcfsvZ+8Cd84HXFjHhV6ZdRPbBWFWd8TlDufrWNuY4SyOtkyfE5E +lWcoRnkBGBaooZNcd72ZwZVO0gAS31bgs8ju+k02GIDLkuZSRNPeFsRQbcMu1Mhc +cFteFsqniLhzpkWJVQQgeGlV -----END CERTIFICATE----- diff --git a/Lib/test/certdata/keycert3.pem b/Lib/test/certdata/keycert3.pem index 20d9bd14e96..097dc9f8e88 100644 --- a/Lib/test/certdata/keycert3.pem +++ b/Lib/test/certdata/keycert3.pem @@ -1,42 +1,42 @@ -----BEGIN PRIVATE KEY----- -MIIG/gIBADANBgkqhkiG9w0BAQEFAASCBugwggbkAgEAAoIBgQCgKihxC+2g7d7M -JfIUBfFWiuMwxg0WhdxPyGUzMAjexbEOHo0ojntxPdH9KYRwiKRKb9jnmzXp2CKT -hqBYJIetq/3LYZp4bvDJ/hVCL9e4jqu1l+wd9DkqhKZ69b6C1/d12JAKvC5TIT+/ -b7EglYU8KMNx2WO5KxIJeVpX68jn49YtUzg0hT0QiXj4eugbDk1L1f99xgvkHaVW -VQxi6MFNWHJq/xXUb8E/hd/Q3oadN1BXMWl9P46D4R+YGKQQdZFkwEJsbqijFvWW -bOoaz7TFxf8n0q616803aXLfaWikfEXLnznEvKo7vyEivtT/y14Nm+SiR3nS6E0y -Dt8gmeHdaHcrmQT+yQ6yNOYDCcfeYM+rBuvOUHPudjMy0k8K/0IPjDo0KActKPH0 -UVbyDBMKDdGQ2+LhRFLsGXHlD9b05PxhqTULe3LeK6KZ+iuGbWtwVLaL5S42WiCA -YXNShE1Ko0Q7wugAippXCf+aWP3Wx9ZTrsfiDBbIfnY5mlfdG90CAwEAAQKCAYAA -ogoE4FoxD5+YyPGa+KcKg4QAVlgI5cCIJC+aMy9lyfw4JRDDv0RnnynsSTS3ySJ1 -FNoTmD5vTSZd1ONfVc2fdxWKrzkQDsgu1C07VLsShKXTEuWg/K0ZKOsLg1scY0Qc -GB4BnNrGA1SgKg3WJiEfqr2S/pvxSGVK2krsHAdwOytGhJStSHWEUjbDLKEsMjNG -AHOBCL5VSXS00aM55NeWuanCGH36l/J4kMvgpHB9wJE1twFGuHCUvtgEHtzPH9fQ -plmI0QDREm6UE6Qh01lxmwx3Xc5ASBURmxs+bxpk94BPRpj8/eF2HPiJalrkJksj -Xk3QQ7k23v6XnmHKV3QqpjUgJTdbuMoTrVMu14cIH6FtXfwVhtthPnCI8rk5Lh8N -cqLC7HT+NE1JyygzuMToOHMmSJTQ8L6BTIaRCZjvGTPYaZfFgeMHvvhAJtP5zAcc -xQzyCyNBU8RdPGT8tJTyDUIRs20poqe7dKrPEIocKJX7tvNSI2QxkQ96Adxo1gEC -gcEAvI8m6QCDGgDWI8yTH9EvZQwq+tF8I+WMC+jbPuDwKg5ZKC7VjRO//9JzPY+c -TxmLnQu64OkECHbu7pswDBbtnPMbodF9inYEY5RkfufEjEMJGEdkoBJWnNx78EkV -bcffWik0wXwdt6jd1CAnjmS9qaPz0T1NV8m5rQQn5JUYXlC9eB2kOojZYLbZBl3g -xUSRbIqHC7h8HuyAU26EPiprHsIxrOpbxABFOdvo2optr50U7X10Eqb4mRQ4z22W -ojJdAoHBANlzJjjEgGVB9W50pJqkTw8wXiTUG8AmhqrVvqEttLPcWpK6QwRkRC+i -5N1iUObf/kOlun2gNfHF6gM68Ja9wb2eGvE5sApq9hPpyYF0LS3g8BbJ9GOs6NU9 -BfM1CkPrDCdc4kzlUpDibvc6Fc9raCqvrZRlKEukqQS8dumVdb74IaPsP6q8sZMz -jibOk0eUrbx2c5vEnd0W8zMeNCuCwO1oXbfenPp/GLX9ZRlolWS/3cQoZYOSQc9J -lFQYkxL3gQKBwQCy3Pwk9AZoqTh4dvtsqArUSImQqRygFIQXXAh1ifxneHrcYijS -jVSIwEHuuIamhe3oyBK6fG8F9IPLtUwLe8hkJDwm8Misiiy5pS77LrFD9+btr/Nk -4GBmpcOveDQqkflt1j6j9y9dY4MhUGsVaLx86fhDmGoAh2tpEtMgwsl91gsUoNGD -cQL6+he+MVkg510nX/Sgipy63M8R1Xj+W1CHueBTTXBE6ZjBPLiSbdOETXZnnaR4 -eQjCdOs64JKOQ0UCgcBZ4kFAYel48aTT/Z801QphCus/afX2nXY5E5Vy5oO1fTZr -RFcDb7bHwhu8bzFl3d0qdUz7NMhXoimzIB/nD5UQHlSgtenQxJnnbVIAEtfCCSL1 -KJG+yfCMhGb7O0d8/6HMe5aHlptkjFS2GOp/DLTIQEoN9yqK6gt7i7PTphY/1C2D -ptpCZzE32a2+2NEEW67dIlFzZ/ihNSVeUfPasHezKtricECPQw4h3BZ4RETMmoq+ -1LvxgPl3B8EqaeYRhwECgcEAjjp/0hu/ukQhiNeR5a9p1ECBFP8qFh6Cpo0Az/DT -1kX0qU8tnT3cYYhwbVGwLxn2HVRdLrbjMj/t88W/LM2IaQ162m7TvvBMxNmr058y -sW/LADp5YWWsY70EJ8AfaTmdQriqKsNiLLpNdgcm1bkwHJ1CNlvEpDs1OOI3cCGi -BEuUmeKxpRhwCaZeaR5tREmbD70My+BMDTDLfrXoKqzl4JrRua4jFTpHeZaFdkkh -gDq3K6+KpVREQFEhyOtIB2kk +MIIG/gIBADANBgkqhkiG9w0BAQEFAASCBugwggbkAgEAAoIBgQDuNnCqYWoguzKI +B3PwOsoRzI8wgNdxx3RiIJSItFISlAgB4XkYzPBpi5nFR8GCqq7IUQQ+iYwdZa3y +H6X11IdlrCx4gDdQFSFOegnG41oqv7GwBEFvq6xlowpoaGw+5/8/sJX/VU44rL9/ +fiMVTbehMhMS/z5nTSYSACV7sRXnwW4T6l2Od77FzPBjnaFFDNr5GIoiD4mNSJ35 +t05tliGcz0IyE5TGR6kIEFNIH5theHoJJ4JC4uLyWJweJpxDx10njWRwKN/tXm8F +nHQPMRAUl2wM07QZEI1yfzUr00/oG2dpPnPyNh+6p2ld8JS0g2/Pd2pQL8OtmM2c +DcN5E+qCEW1UVIJ5//ad8pUcoQwyRIBROJbakqv6FcVTdgTogoq5Vu+T6TVFLSIS +vVvpsV0MtT/H9/9TtCnqOwliE8s8kb3NjQT8e72VHNd43bVPE4DeQ+XlbMoKzV+G +OmSHSfzSOZB9kScSmGd5YD8783TbL8/7Nn1ERQdWqaixGI36+BECAwEAAQKCAYBV +Ubhevg9V89ZwdELpSxUu9NZgZ/VCck7UCplIsVUoBE8t5UULRfPhybdkuoOrulhp +tOLRR1ChAtcffohhmSJ5nwY6jqnBDCBmzD0OOEYGQ6xvv8Z0KcfQi2nh5WzHxy5b +8HJ5BmPC1tSr5FDKg5B+ssG0Lyl5tF8rWVQTjmSrIlO73Fhv+6GflPyQoVeutKEF +UO3Ar1H0AYtbcnUruPcHBBDQgMTrk6UOF1LM5U0wxwbmmnkEXeEtLeGUxv13JUMe +QTrSw0P+hn5uiDMwY5lI212ayorbwyNuzU0hGW7j98qa/S9MVSeRwv4bsbtBqwtT +ZVGL2TogjeJ+8+qDlv5tf3crD1pV0yu7uVUeIGrJZJdiiMM6N7/x3cIJtrLDaF1w +9kUqdQGWDWwRV3w0FaeXDVlUTJ3VFTZIinLo1Vmd3AXLn97eVy+68DWD6ELlMkfF +ro3bQbQu59DGswDF9+mMauhgvljnxY/wUnkyp8LGZt8w/RU/op8LcvPKFjQfX6UC +gcEA/RClguU/JTWQSF4Vjy5+DmrzSCXabrO7986lkOwPXEaXLsF2pGOU+sbxffXZ +PCQ4zU6aNYBMtNWLrVqBrPnI78XZQbmSG7zUDxWwmr1ttuSHGthdaZUVTOsFsnpE +R+1lj20G2T2ankzv0ICFz7BWyaYwyzDrABwuP189b3juuX6xGLlCO25/AnK+iTXQ +WF4ZURZiwDg4WpxvPsE2FFglN09TtDDCeQw/zv7Jh6jWtHZfivfv/wR8ILu72gde +LdinAoHBAPD5slojp2St6iWTaxcYYBc+4x+Rv7oyxeAyM/00iIAnf19oaAw3jki1 +yz6zeNnhaZuwa6ruvhJAESPc+pXkmoPpQ1yIRjKZc3jPVoaxYuSPgUEbHV6BQbjD +clIoGkC6ISsjanMq8Z3pK7e2bxY4mxGRQIFSFnTeK8m8nzq5dYisVtxxJFnV2fDG +NWDMdDYBh52vpkPjnm5OoOuilkEH2ygtZlT6bKAbnhi/FVjYCHk46inaYx8PakMu +LnvZI0OIhwKBwBGzYGBPeKM5o+Xr7sYdEmQfxvR88VJc8ADdS2dfm5NwvJJgpdPJ +w1nnIG0XDSLPxclWfiLP3o2ngiWV9wwKTKu4wwF94WJfStXjRn8MUOhCA9E04RPJ +gbvnlHZvZudBC6GElr4LOQ1phDypQLLOOsPQBAmyWj2fuvxjxQBPDSOcYPbBvog5 +qliZfgpK4U/NBShO0IlxZT+xQXa6PPYfVDsSKWCpKHEfEjeASshaXuowfW5S+U51 +GdmQSAtwCH5ccQKBwQCOLgW5gYfms2aPvSdWfR9VF9nSaqCBMCvoWDasky5mzucs +V+HsM2tUI09EM4h+pa02GyWruSmUgxCZ5GxFvJgedKc2FYG1oSysf0lCN69tw+4z +h9gQRpuMdGUjbF3xCuE/HqpUQWZGEamlv5JTvhpghx9ULibp1Zxob05Ty9E5TtYB +QxB7oN3yXkBoWLnIk6Z8t4KWU9rKosH3xfp5bDU2w3K5ePhWj3T8jOH/hZeaTqZ7 +A0uwq9u6v6jVkgxocEkCgcEAhyMPyiMqqq55qD6JlPY4zf4mWQJS4MnmWexH/5N1 +a6uogerwZnIjuH0/wNc2qzQSOwVJRqjlxRZbEkWST0JK/meXZ+e0nqaROWICutO6 +ciIf2wVjTzn1f85DHwPhUf3P2zIfpVFh+XuvSb32J+pzNwK40gMhirYMDSbsm/RQ +JrJHQwEX9BvI6IP5kaMOMlSwiG3Soo3MxeJWmozkz09+cs7BwMFsUKOdqruJHSaw +/Q06DQ9u9UyjmLyKv3IkN/cK -----END PRIVATE KEY----- Certificate: Data: @@ -47,38 +47,38 @@ Certificate: Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server Validity Not Before: Aug 29 14:23:16 2018 GMT - Not After : Oct 28 14:23:16 2037 GMT + Not After : Oct 28 14:23:16 2525 GMT Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=localhost Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (3072 bit) Modulus: - 00:a0:2a:28:71:0b:ed:a0:ed:de:cc:25:f2:14:05: - f1:56:8a:e3:30:c6:0d:16:85:dc:4f:c8:65:33:30: - 08:de:c5:b1:0e:1e:8d:28:8e:7b:71:3d:d1:fd:29: - 84:70:88:a4:4a:6f:d8:e7:9b:35:e9:d8:22:93:86: - a0:58:24:87:ad:ab:fd:cb:61:9a:78:6e:f0:c9:fe: - 15:42:2f:d7:b8:8e:ab:b5:97:ec:1d:f4:39:2a:84: - a6:7a:f5:be:82:d7:f7:75:d8:90:0a:bc:2e:53:21: - 3f:bf:6f:b1:20:95:85:3c:28:c3:71:d9:63:b9:2b: - 12:09:79:5a:57:eb:c8:e7:e3:d6:2d:53:38:34:85: - 3d:10:89:78:f8:7a:e8:1b:0e:4d:4b:d5:ff:7d:c6: - 0b:e4:1d:a5:56:55:0c:62:e8:c1:4d:58:72:6a:ff: - 15:d4:6f:c1:3f:85:df:d0:de:86:9d:37:50:57:31: - 69:7d:3f:8e:83:e1:1f:98:18:a4:10:75:91:64:c0: - 42:6c:6e:a8:a3:16:f5:96:6c:ea:1a:cf:b4:c5:c5: - ff:27:d2:ae:b5:eb:cd:37:69:72:df:69:68:a4:7c: - 45:cb:9f:39:c4:bc:aa:3b:bf:21:22:be:d4:ff:cb: - 5e:0d:9b:e4:a2:47:79:d2:e8:4d:32:0e:df:20:99: - e1:dd:68:77:2b:99:04:fe:c9:0e:b2:34:e6:03:09: - c7:de:60:cf:ab:06:eb:ce:50:73:ee:76:33:32:d2: - 4f:0a:ff:42:0f:8c:3a:34:28:07:2d:28:f1:f4:51: - 56:f2:0c:13:0a:0d:d1:90:db:e2:e1:44:52:ec:19: - 71:e5:0f:d6:f4:e4:fc:61:a9:35:0b:7b:72:de:2b: - a2:99:fa:2b:86:6d:6b:70:54:b6:8b:e5:2e:36:5a: - 20:80:61:73:52:84:4d:4a:a3:44:3b:c2:e8:00:8a: - 9a:57:09:ff:9a:58:fd:d6:c7:d6:53:ae:c7:e2:0c: - 16:c8:7e:76:39:9a:57:dd:1b:dd + 00:ee:36:70:aa:61:6a:20:bb:32:88:07:73:f0:3a: + ca:11:cc:8f:30:80:d7:71:c7:74:62:20:94:88:b4: + 52:12:94:08:01:e1:79:18:cc:f0:69:8b:99:c5:47: + c1:82:aa:ae:c8:51:04:3e:89:8c:1d:65:ad:f2:1f: + a5:f5:d4:87:65:ac:2c:78:80:37:50:15:21:4e:7a: + 09:c6:e3:5a:2a:bf:b1:b0:04:41:6f:ab:ac:65:a3: + 0a:68:68:6c:3e:e7:ff:3f:b0:95:ff:55:4e:38:ac: + bf:7f:7e:23:15:4d:b7:a1:32:13:12:ff:3e:67:4d: + 26:12:00:25:7b:b1:15:e7:c1:6e:13:ea:5d:8e:77: + be:c5:cc:f0:63:9d:a1:45:0c:da:f9:18:8a:22:0f: + 89:8d:48:9d:f9:b7:4e:6d:96:21:9c:cf:42:32:13: + 94:c6:47:a9:08:10:53:48:1f:9b:61:78:7a:09:27: + 82:42:e2:e2:f2:58:9c:1e:26:9c:43:c7:5d:27:8d: + 64:70:28:df:ed:5e:6f:05:9c:74:0f:31:10:14:97: + 6c:0c:d3:b4:19:10:8d:72:7f:35:2b:d3:4f:e8:1b: + 67:69:3e:73:f2:36:1f:ba:a7:69:5d:f0:94:b4:83: + 6f:cf:77:6a:50:2f:c3:ad:98:cd:9c:0d:c3:79:13: + ea:82:11:6d:54:54:82:79:ff:f6:9d:f2:95:1c:a1: + 0c:32:44:80:51:38:96:da:92:ab:fa:15:c5:53:76: + 04:e8:82:8a:b9:56:ef:93:e9:35:45:2d:22:12:bd: + 5b:e9:b1:5d:0c:b5:3f:c7:f7:ff:53:b4:29:ea:3b: + 09:62:13:cb:3c:91:bd:cd:8d:04:fc:7b:bd:95:1c: + d7:78:dd:b5:4f:13:80:de:43:e5:e5:6c:ca:0a:cd: + 5f:86:3a:64:87:49:fc:d2:39:90:7d:91:27:12:98: + 67:79:60:3f:3b:f3:74:db:2f:cf:fb:36:7d:44:45: + 07:56:a9:a8:b1:18:8d:fa:f8:11 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Alternative Name: @@ -90,9 +90,9 @@ Certificate: X509v3 Basic Constraints: critical CA:FALSE X509v3 Subject Key Identifier: - 3F:B1:E9:4F:A0:BE:30:66:3E:0A:18:C8:0F:47:1A:4F:34:6A:0F:42 + 16:22:C9:AA:20:D4:F4:8E:ED:9C:A6:BB:23:F5:F2:29:44:ED:F2:1E X509v3 Authority Key Identifier: - keyid:F3:EC:94:8E:F2:8E:30:C4:8E:68:C2:BF:8E:6A:19:C0:C1:9F:76:65 + keyid:C0:0A:2B:28:43:DE:5F:C9:7D:47:E5:47:9B:36:F2:65:8C:67:3B:E2 DirName:/C=XY/O=Python Software Foundation CA/CN=our-ca-server serial:CB:2D:80:99:5A:69:52:5B Authority Information Access: @@ -103,59 +103,59 @@ Certificate: URI:http://testca.pythontest.net/testca/revocation.crl Signature Algorithm: sha256WithRSAEncryption Signature Value: - ca:34:ba:c5:d0:cf:27:31:32:d6:0d:27:30:b8:db:17:df:b7: - 39:a7:bb:b1:3b:86:c4:31:fd:fb:ab:db:63:1a:cc:90:ab:b9: - 4e:ab:34:49:0c:5e:8c:3e:70:a3:a7:6b:2f:a7:9a:25:7b:01: - 5a:18:96:48:76:f8:36:78:74:fa:bc:7d:68:7f:e5:ca:a6:9d: - 7b:dc:72:bd:a3:25:51:17:68:e8:e9:d7:02:86:2c:7d:16:7c: - b5:dc:44:b2:0a:e3:f7:c9:33:a3:51:36:83:bc:d4:70:cd:84: - 91:9f:06:ba:2d:d2:05:0a:65:c3:d9:55:09:a8:b8:09:69:bb: - 93:86:c2:b7:c2:90:74:7c:bf:f0:5d:bc:0e:63:13:8c:eb:fa: - 0f:f1:fa:e5:12:70:4d:0c:eb:8c:2e:a2:42:42:00:04:0f:fc: - f9:1f:41:9c:63:78:f0:66:93:b2:8f:2e:e8:93:1c:50:cb:2d: - 7f:b6:ba:57:6f:52:62:d7:39:0b:09:82:ab:a6:53:4d:cc:05: - 3e:19:f0:d4:c0:ce:a9:ad:10:ce:b9:71:e4:8f:f2:5a:3c:65: - ba:dc:cb:e0:04:90:2b:a5:15:a6:7d:da:dc:a3:b5:b7:bc:a0: - de:30:4e:64:cb:17:0d:3a:a0:52:d2:67:3b:a2:3a:00:d5:39: - aa:61:75:52:9f:fe:9b:c0:e8:a0:69:af:a8:b3:a3:1d:0a:40: - 52:04:e7:3d:c0:00:96:5f:2b:33:06:0c:30:f6:d3:18:72:ee: - 38:d0:64:d3:00:86:37:ec:4f:e9:38:49:e6:01:ff:a2:9a:7c: - dc:6a:d3:cb:a8:ba:58:fb:c3:86:78:47:f1:06:a6:45:e7:53: - de:99:1d:81:e6:bc:63:74:46:7c:70:23:57:29:60:70:9a:cc: - 6f:00:8e:c2:bf:6a:73:7d:6e:b0:62:e6:dc:13:1a:b9:fe:0f: - c2:d1:06:a1:79:62:7f:b6:30:a9:03:d0:47:57:25:db:48:10: - d1:cf:fb:7d:ac:3d + 43:82:1f:18:b4:cb:7a:b7:4e:ac:c3:65:2e:b7:57:3a:f0:78: + b3:b2:20:3e:bd:7b:24:f7:e7:17:cc:62:c0:b8:63:f0:2e:30: + 0c:54:be:c9:94:7c:54:d4:40:06:6e:92:9d:72:2c:44:f5:e3: + c1:4a:3f:26:e4:21:ca:ba:ef:4f:a3:50:56:ee:89:5c:dc:5b: + 98:bc:0c:e9:db:16:98:f0:4c:3f:65:b9:d9:e0:2e:bd:83:9e: + c7:1b:f7:4f:87:5f:75:ff:67:5d:0a:ff:ce:fa:0b:f1:d5:8b: + 58:23:2b:92:27:b8:15:ff:66:c2:f9:68:e2:ed:64:a0:4a:e0: + 42:8b:ac:35:6a:67:64:f5:cb:a9:00:37:20:56:dd:3a:02:6e: + d8:0f:b4:5d:ff:92:a3:0d:65:fc:2e:ab:4e:37:bc:7e:d6:5b: + 1c:55:78:a0:6e:a9:2e:55:31:cf:61:f0:1b:4c:fe:fe:60:75: + 76:e8:d9:a1:f5:14:f0:43:33:4d:52:05:be:33:ff:da:31:94: + bd:0d:e5:4d:a8:ee:d1:f3:92:34:c2:36:91:37:3a:6d:ec:20: + df:b3:9e:a2:5d:d0:f0:a1:e2:ff:b7:85:ce:09:e3:c8:4a:6a: + 93:a4:ae:ec:c1:67:42:f7:8e:2f:64:ca:d7:76:72:bf:3f:ef: + 64:ff:10:dd:a2:3e:2d:f5:aa:02:cf:31:f0:6d:93:6b:56:4c: + b0:88:95:54:dc:d4:ac:dc:c1:88:27:8b:d9:9c:4e:66:93:f8: + dd:5e:74:41:26:9a:ef:e8:c8:46:5c:d5:f4:fc:cd:cc:67:c7: + 33:62:2e:70:9b:d1:39:de:34:fd:ad:9d:b9:48:61:e7:89:53: + 62:62:f0:40:6c:68:2f:e1:da:40:61:2a:50:af:c9:41:6d:98: + 16:af:cb:63:03:ec:04:99:e0:39:98:60:ae:aa:6a:57:9b:25: + 73:03:16:bf:78:c8:d7:2f:eb:fc:64:7f:c7:78:8c:34:15:5b: + 66:40:98:86:e6:0a -----BEGIN CERTIFICATE----- -MIIF8TCCBFmgAwIBAgIJAMstgJlaaVJcMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +MIIF8zCCBFugAwIBAgIJAMstgJlaaVJcMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW -MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODA4MjkxNDIzMTZaFw0zNzEwMjgx -NDIzMTZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEj -MCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMMCWxv -Y2FsaG9zdDCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAKAqKHEL7aDt -3swl8hQF8VaK4zDGDRaF3E/IZTMwCN7FsQ4ejSiOe3E90f0phHCIpEpv2OebNenY -IpOGoFgkh62r/cthmnhu8Mn+FUIv17iOq7WX7B30OSqEpnr1voLX93XYkAq8LlMh -P79vsSCVhTwow3HZY7krEgl5WlfryOfj1i1TODSFPRCJePh66BsOTUvV/33GC+Qd -pVZVDGLowU1Ycmr/FdRvwT+F39Dehp03UFcxaX0/joPhH5gYpBB1kWTAQmxuqKMW -9ZZs6hrPtMXF/yfSrrXrzTdpct9paKR8RcufOcS8qju/ISK+1P/LXg2b5KJHedLo -TTIO3yCZ4d1odyuZBP7JDrI05gMJx95gz6sG685Qc+52MzLSTwr/Qg+MOjQoBy0o -8fRRVvIMEwoN0ZDb4uFEUuwZceUP1vTk/GGpNQt7ct4ropn6K4Zta3BUtovlLjZa -IIBhc1KETUqjRDvC6ACKmlcJ/5pY/dbH1lOux+IMFsh+djmaV90b3QIDAQABo4IB -wDCCAbwwFAYDVR0RBA0wC4IJbG9jYWxob3N0MA4GA1UdDwEB/wQEAwIFoDAdBgNV -HSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4E -FgQUP7HpT6C+MGY+ChjID0caTzRqD0IwfQYDVR0jBHYwdIAU8+yUjvKOMMSOaMK/ -jmoZwMGfdmWhUaRPME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29m -dHdhcmUgRm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcoIJAMst -gJlaaVJbMIGDBggrBgEFBQcBAQR3MHUwPAYIKwYBBQUHMAKGMGh0dHA6Ly90ZXN0 -Y2EucHl0aG9udGVzdC5uZXQvdGVzdGNhL3B5Y2FjZXJ0LmNlcjA1BggrBgEFBQcw -AYYpaHR0cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0Y2Evb2NzcC8wQwYD -VR0fBDwwOjA4oDagNIYyaHR0cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0 -Y2EvcmV2b2NhdGlvbi5jcmwwDQYJKoZIhvcNAQELBQADggGBAMo0usXQzycxMtYN -JzC42xfftzmnu7E7hsQx/fur22MazJCruU6rNEkMXow+cKOnay+nmiV7AVoYlkh2 -+DZ4dPq8fWh/5cqmnXvccr2jJVEXaOjp1wKGLH0WfLXcRLIK4/fJM6NRNoO81HDN -hJGfBrot0gUKZcPZVQmouAlpu5OGwrfCkHR8v/BdvA5jE4zr+g/x+uUScE0M64wu -okJCAAQP/PkfQZxjePBmk7KPLuiTHFDLLX+2uldvUmLXOQsJgqumU03MBT4Z8NTA -zqmtEM65ceSP8lo8Zbrcy+AEkCulFaZ92tyjtbe8oN4wTmTLFw06oFLSZzuiOgDV -OaphdVKf/pvA6KBpr6izox0KQFIE5z3AAJZfKzMGDDD20xhy7jjQZNMAhjfsT+k4 -SeYB/6KafNxq08uoulj7w4Z4R/EGpkXnU96ZHYHmvGN0RnxwI1cpYHCazG8AjsK/ -anN9brBi5twTGrn+D8LRBqF5Yn+2MKkD0EdXJdtIENHP+32sPQ== +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAgFw0xODA4MjkxNDIzMTZaGA8yNTI1MTAy +ODE0MjMxNlowXzELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4 +MSMwIQYDVQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjESMBAGA1UEAwwJ +bG9jYWxob3N0MIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEA7jZwqmFq +ILsyiAdz8DrKEcyPMIDXccd0YiCUiLRSEpQIAeF5GMzwaYuZxUfBgqquyFEEPomM +HWWt8h+l9dSHZawseIA3UBUhTnoJxuNaKr+xsARBb6usZaMKaGhsPuf/P7CV/1VO +OKy/f34jFU23oTITEv8+Z00mEgAle7EV58FuE+pdjne+xczwY52hRQza+RiKIg+J +jUid+bdObZYhnM9CMhOUxkepCBBTSB+bYXh6CSeCQuLi8licHiacQ8ddJ41kcCjf +7V5vBZx0DzEQFJdsDNO0GRCNcn81K9NP6BtnaT5z8jYfuqdpXfCUtINvz3dqUC/D +rZjNnA3DeRPqghFtVFSCef/2nfKVHKEMMkSAUTiW2pKr+hXFU3YE6IKKuVbvk+k1 +RS0iEr1b6bFdDLU/x/f/U7Qp6jsJYhPLPJG9zY0E/Hu9lRzXeN21TxOA3kPl5WzK +Cs1fhjpkh0n80jmQfZEnEphneWA/O/N02y/P+zZ9REUHVqmosRiN+vgRAgMBAAGj +ggHAMIIBvDAUBgNVHREEDTALgglsb2NhbGhvc3QwDgYDVR0PAQH/BAQDAgWgMB0G +A1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMBAf8EAjAAMB0GA1Ud +DgQWBBQWIsmqINT0ju2cprsj9fIpRO3yHjB9BgNVHSMEdjB0gBTACisoQ95fyX1H +5UebNvJljGc74qFRpE8wTTELMAkGA1UEBhMCWFkxJjAkBgNVBAoMHVB5dGhvbiBT +b2Z0d2FyZSBGb3VuZGF0aW9uIENBMRYwFAYDVQQDDA1vdXItY2Etc2VydmVyggkA +yy2AmVppUlswgYMGCCsGAQUFBwEBBHcwdTA8BggrBgEFBQcwAoYwaHR0cDovL3Rl +c3RjYS5weXRob250ZXN0Lm5ldC90ZXN0Y2EvcHljYWNlcnQuY2VyMDUGCCsGAQUF +BzABhilodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rlc3RjYS9vY3NwLzBD +BgNVHR8EPDA6MDigNqA0hjJodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rl +c3RjYS9yZXZvY2F0aW9uLmNybDANBgkqhkiG9w0BAQsFAAOCAYEAQ4IfGLTLerdO +rMNlLrdXOvB4s7IgPr17JPfnF8xiwLhj8C4wDFS+yZR8VNRABm6SnXIsRPXjwUo/ +JuQhyrrvT6NQVu6JXNxbmLwM6dsWmPBMP2W52eAuvYOexxv3T4dfdf9nXQr/zvoL +8dWLWCMrkie4Ff9mwvlo4u1koErgQousNWpnZPXLqQA3IFbdOgJu2A+0Xf+Sow1l +/C6rTje8ftZbHFV4oG6pLlUxz2HwG0z+/mB1dujZofUU8EMzTVIFvjP/2jGUvQ3l +Taju0fOSNMI2kTc6bewg37Oeol3Q8KHi/7eFzgnjyEpqk6Su7MFnQveOL2TK13Zy +vz/vZP8Q3aI+LfWqAs8x8G2Ta1ZMsIiVVNzUrNzBiCeL2ZxOZpP43V50QSaa7+jI +RlzV9PzNzGfHM2IucJvROd40/a2duUhh54lTYmLwQGxoL+HaQGEqUK/JQW2YFq/L +YwPsBJngOZhgrqpqV5slcwMWv3jI1y/r/GR/x3iMNBVbZkCYhuYK -----END CERTIFICATE----- diff --git a/Lib/test/certdata/keycert3.pem.reference b/Lib/test/certdata/keycert3.pem.reference new file mode 100644 index 00000000000..84d2ca29953 --- /dev/null +++ b/Lib/test/certdata/keycert3.pem.reference @@ -0,0 +1,15 @@ +{'OCSP': ('http://testca.pythontest.net/testca/ocsp/',), + 'caIssuers': ('http://testca.pythontest.net/testca/pycacert.cer',), + 'crlDistributionPoints': ('http://testca.pythontest.net/testca/revocation.crl',), + 'issuer': ((('countryName', 'XY'),), + (('organizationName', 'Python Software Foundation CA'),), + (('commonName', 'our-ca-server'),)), + 'notAfter': 'Oct 28 14:23:16 2525 GMT', + 'notBefore': 'Aug 29 14:23:16 2018 GMT', + 'serialNumber': 'CB2D80995A69525C', + 'subject': ((('countryName', 'XY'),), + (('localityName', 'Castle Anthrax'),), + (('organizationName', 'Python Software Foundation'),), + (('commonName', 'localhost'),)), + 'subjectAltName': (('DNS', 'localhost'),), + 'version': 3} diff --git a/Lib/test/certdata/keycert4.pem b/Lib/test/certdata/keycert4.pem index ff4dceac790..fcaa3a1656d 100644 --- a/Lib/test/certdata/keycert4.pem +++ b/Lib/test/certdata/keycert4.pem @@ -1,42 +1,42 @@ -----BEGIN PRIVATE KEY----- -MIIG/QIBADANBgkqhkiG9w0BAQEFAASCBucwggbjAgEAAoIBgQDGKA1zZDjeNPh2 -J9WHVXXMUf8h5N4/bHCM3CbIaZ1dShkCgfmFWmOtruEihgbfRYaSWZAwCmVAQGjm -gvUfgOIgsFfM8yO+zDByPhza7XvWPZfEe7mNRFe5ZlYntbeM/vuWCM4VzwDq/mqF -TFxNRmwInqE7hx0WnfCoQWe9N41hJyl1K0OjADb+SjlpJ0/UJ63hsB+dowGjaaBv -J8HduQcRqNg8s6FcyJJ8Mjss1uRMFK2j9QrmgbA61XuIPCxzc3J57mW8FN2KsR8D -2HOhe9nsTGlxp+O5Cudf/RBWB443xcoyduwRXOFTdEAU45MS4tKGP2hzezuxMFQn -LKARXVW4/gFxZk7kU8TweZUS6LAYPfYJnlfteb6z37LAbtoDvzKUKBEDf/nmoa7C -uKxSPC5HIKhLbjU/6kuPglSVEfJPJWu2bZJDAkFL85Ot3gPs10EX2lMUy0Jt3tf+ -TaQjEvFZhpKN8KAdYj3eVgOfzIBbQyjotHJjFe9Jkq4q7RoI+ncCAwEAAQKCAYAH -tRsdRh1Z7JmHOasy+tPDsvhVuWLHMaYlScvAYhJh/W65YSKd56+zFKINlX3fYcp5 -Fz67Yy+uWahXVE2QgFou3KX0u+9ucRiLFXfYheWL3xSMXJgRee0LI/T7tRe7uAHu -CnoURqKCulIqzLOO1efx1eKasXmVuhEtmjhVpcmDGv8SChSKTIjzgOjqT7QGE9Xq -eSRhq7mulpq9zWq+/369yG+0SvPs60vTxNovDIaBn/RHSW5FjeDss5QnmYMh/ukN -dggoKllQlkTzHSxHmKrIJuryZC+bsqvEPUFXN0NMUYcZRvt1lwdjzq/A+w4gDDZG -7QqAzYMYQZMw9PJeHqu4mxfUX5hJWuAwG5I2eV3kBRheoFw7MxP0tw40fPlFU+Zh -pRXbKwhMAlIHi0D8NyMn3hkVPyToWVVY3vHRknBB/52RqRq3MjqEFaAZfp0nFkiF -ytv3Dd5aeBb1vraOIREyhxIxE/qY8CtZC+6JI8CpufLmFXB412WPwl0OrVpWYfEC -gcEA486zOI46xRDgDw0jqTpOFHzh+3VZ8UoPoiqCjKzJGnrh2EeLvTsXX/GZOj0m -5zl6RHEGFjm5vKCh2C72Vj/m+AFVy7V9iJRzTYzP8So/3paaqo7ZaROTa6uStxdD -VPnY1uIgVQz9w5coN4dmr+RLBpFvvWaHp1wuC08YIWxcC9HSTQpbi1EP5eo08fOk -8reNkDEHxihDGHr1xW0z0qJqK1IVyLP7wDkmapudMZjkjqJSGJwwefV4qyGMTV2b -suW1AoHBAN6t9n6LBH553MF5iUrNJYxXh/SCom4Zft9aD6W4bZV/xL4XPpKBB4HX -aWdeI0iYZU9U+CZ88tBoQCt+JMrJ9cz03ENOvA/MBMREwbZ2hKmQgnoDZsV0vNry -6UsxeQmeNpGQFUz9foVJQVRdQCceN2YEABdehV1HZoSBbuGZkzqGJXrWwaf/ZhpB -dPYGUGOsczoD2/QLuWy2M7f7v0Ews6Heww3zipWzvdxKE0IpyVs30ZwVi8CRQiWU -bEcleXP6+wKBwAi3xEwJxV39Q1XQHuk+/fXywYMp/oMpXmfKUKypgBivUy0/r61S -MZbOXBrKdE6s+GzeFmmLU/xP+WGYinzKfUBIbMwa6e7sH218UgjcoQ0Xnlugk9ld -kmqwajDvhvgdh5rRlIMsuBlgE33shJV+mxBpSGlrHw3cjTaJlFbTGsKpCO9B0jcG -pyEZUWVg+ZMASz6VYcLHj6nEKtufTjhlVsLJpWPE34F/rmSuB9n6C+UZeSLP91rz -dea2pfPf/TFfcQKBwF4DSj9Qx/vxzS7t9fXbuM+QoPitMpCTOQppRpPr0nA8uj6b -J7LIwPejj3+xsenTVWpx8DanqAgvC3CRWE05iQoYEupj0mhE9Xo7oSE81nOUbFHB -H+GbkKRLzA0P/Q7/egBouWWA3Kq/K9LHb+9UBYWPiM5U/K9OFs04rCyZHxylSCud -gbNA08Wf/xZjwgri4t8KhBF75bQtFJbHtY57Vkuv9d/tA4SCl1Tq/UiAxd86KMfi -HNeXPDsLd89t1eIOgwKBwQDJkqwZXkhwkhoNuHRdzPO/1f5FyKpQxFs+x+OBulzG -zuwVKIawsLlUR4TBtF7PChOSZSH50VZaBI5kVti79kEtfNjfAzg4kleHrY8jQ/eq -HludZ3nmiPqqlbH4MH8NWczPEjee6z4ODROsAe31pz3S8YQK7KVoEuSf0+usJ894 -FtzS5wl6POAXTo2QeSNg9zTbb6JjVYcq6KCTnflDm4YEvFKI+ARqAXQHxm05wEOe -DbKC6hxxQbDaNOvXEAda8wU= +MIIG/gIBADANBgkqhkiG9w0BAQEFAASCBugwggbkAgEAAoIBgQC/uCI/4eVSCIrx +//GcCCxeQP16Mtf0wAuINBR9uTySF8sEVoS6rNBwLeudZXaHmcp3jM7N2e9GjI8d +dQccPApKBwjAxhdT4XeuUTep4VraKWjLTWDvABHfCVFB1tX3HusEWaS8n/pH+N4L +UfnxS3HnFqGC5ib2DSgFTCnLQSj82iUgACdf87iZTons+kRvqYsdwpsgBUnO+w+o +c5o7Nj1bJJlTREsY6kQGY94as4pCkmqhoBTC7/le9wFGmXipRYi3pfSTOqAQJfqW +1VlQ4Bbc3lN5L+ONTmFXOgTJ5qhaS8cglnYElJI1QQTagyWQMHuBDIMpr3Iv/5qU +UzdIMzoMrpX6ien3Gan/jyjjzvP95jJvGujQudw/myXK8VOUThVtstRDFA3cIoV4 +UqMODz8w6qYU0GPYFBq//1UdpDrk/fBPBe3MLJT2+sB/kTzwxxYsKqcdjPG2Bt3Y +2nk7hCzE0KuueqCQPk/3fomY1TDyTlWOriAS40EYue5Rhx9qr8kCAwEAAQKCAYAL +X7EoeQ5Rv4/+q2B6EUIQlWp0RW/qZTpJ6k0M6GBfimnQ6BEXwgjbnt3LiKlvggAw +93mNXNCFLvGOndK+KhGyMpiiVx4rK8Ud4lObEHODXdGJvh1yEF7/DF51uXkYIA1x +RKAxUIxYmLsTkNlzJzaqrv0F9wF4t28YYVxZYpQ76/Un4Np1JtBcx/wGwxIsTbKj +IVhynd2zGdHj/He5643YSmOOPQ73e25tsp8KlnwtVuJclFKm/fWdXKoGtjQiVq2b +bViG7WTqb+a+R3IrO8CE2vAQ1ZqklP4Pg33AU1XFogRweqSkXvBGy1GHSSXCO4uX +gORj/bo/gAZ/eeufwqxBxJMS4Kbmxes/oYEgJidoaeA6DZjcKr+J3rFUt9j6rpxt +abeKBiK6wTHkKAdQwoccDdKFROvL4ZEu4zpjkmiqDrtIZEqtB8YCvoMpteIqrBhr +stLCDrftsysBTNGNn0THb4H5wx8nkZPCVyvaBD4N4Skp5rapP2bbrPhigcOoR8EC +gcEA4ToVg2b09tLWpmCg6XXssT1SvvFcSaJ3jjRKMfpmP5epWFiZ21qAOCbYhMNu +2G21AyNd1W7oVi6epXsB2YBOlSI0Fr6RuN0fDwg5jZLxEkEqDsL2QKsgoKHtGfBn +gX9jLQ5Lsy5/0CRXYCfPjdccbr2J5Gm2fU/jhhCi9ji/6b2RbgrRkIkr2I/J6KbS +sO+p1d93CFNagLMbdDspugDlIgj52OBMYO60nDlNU8O2pkAcLQTpWVqEjmoo4Q4K +oyOXAoHBANnqB93sLCndEiZNth7cOOEhNKCWRDkpGwDBbn2Dj3kBaXg8eCTG7tiF +FPyNVydYg+u0vcPIjj9URsOfpHc9MLSYtKAgCL3rSk+OQnROeg9hCjmB5Ucg7NTR +gSh/flfWp/q1EDeOVAu8B9DuBX1209br4xi78LnBP5/UzRjhuFEmvCvOqVCMCI7Y +I1k3l4NhOk87fKq5dbtxvNTU2eFTtt1+YjYwZFY+AUmx3WBz+aFwygks3S4SvN1J +LtLVYhOznwKBwQDC047WixIuDLX3WDD5orOroeNZHsn5PFv1HBBuaS9XpSatMH9u ++ztc12WGetQAze2+GDLMNNMv8cX0WZKBBfd0FBFA93pwkn6Sb0fxyoFUjCAIguen +iyB/M3M5c/blUz+EMxCSoA+aCkW2/NkS1lhXBwgoGLXuclPbnbqKCQ8h74TEzwD2 +6WGPRNqgsOYifj7IrjR2dDwehlCiW6c9qhaLOX5+94+6beK4HO1iHzN5Xo3A97Wv +QJjX5McV3yKeemMCgcA4N3reEo52IlULUqL4JSH7WkCkaP+iq2sO79fcQ3Ng6S9X +WGo6OqPlcbevS5s/SEOILDGEb5na1pgG4YlhRYTqIjb+1CTNMgUSrwWP0asFiqhD +m7IVfnX6lS23z+Q9LuBY+hr76hjeihyOFsmNy3jtCh+lAt8gXK1YQ2LB14FgVhjX +SFI/uFCA4VuFKaVJvGx5gkQwGvY3bCkl0t9+lMUpMPCPQD6yTP6yD1OoDWNJ9bn5 +UfyhZS4Z/EY7F9dcc8sCgcEAi8yW4gNCZSsGQRLh+fyDCSdak4ahATAeiI24231u +jK5+ncK9ZoZvzO/h+V5bw7ha59rD8aGmrLqzBVKEMi0PAT38psNH9ZtRt9y29YQ8 +WpyHtODMbphfbH4SFH3nj3PldRtBcOt0EDdp/4s64A9qdOWRx/vlJhdg0vtWTy5s +koyyOHhTcplpyPPq0BetYKl9UlXPVjus/RT1XV/lz2YrE8E9UAq+/r7ECoMExaZt +8j8J8O3lTWPxNTaQuiPiHA7E -----END PRIVATE KEY----- Certificate: Data: @@ -47,38 +47,38 @@ Certificate: Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server Validity Not Before: Aug 29 14:23:16 2018 GMT - Not After : Oct 28 14:23:16 2037 GMT + Not After : Oct 28 14:23:16 2525 GMT Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=fakehostname Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (3072 bit) Modulus: - 00:c6:28:0d:73:64:38:de:34:f8:76:27:d5:87:55: - 75:cc:51:ff:21:e4:de:3f:6c:70:8c:dc:26:c8:69: - 9d:5d:4a:19:02:81:f9:85:5a:63:ad:ae:e1:22:86: - 06:df:45:86:92:59:90:30:0a:65:40:40:68:e6:82: - f5:1f:80:e2:20:b0:57:cc:f3:23:be:cc:30:72:3e: - 1c:da:ed:7b:d6:3d:97:c4:7b:b9:8d:44:57:b9:66: - 56:27:b5:b7:8c:fe:fb:96:08:ce:15:cf:00:ea:fe: - 6a:85:4c:5c:4d:46:6c:08:9e:a1:3b:87:1d:16:9d: - f0:a8:41:67:bd:37:8d:61:27:29:75:2b:43:a3:00: - 36:fe:4a:39:69:27:4f:d4:27:ad:e1:b0:1f:9d:a3: - 01:a3:69:a0:6f:27:c1:dd:b9:07:11:a8:d8:3c:b3: - a1:5c:c8:92:7c:32:3b:2c:d6:e4:4c:14:ad:a3:f5: - 0a:e6:81:b0:3a:d5:7b:88:3c:2c:73:73:72:79:ee: - 65:bc:14:dd:8a:b1:1f:03:d8:73:a1:7b:d9:ec:4c: - 69:71:a7:e3:b9:0a:e7:5f:fd:10:56:07:8e:37:c5: - ca:32:76:ec:11:5c:e1:53:74:40:14:e3:93:12:e2: - d2:86:3f:68:73:7b:3b:b1:30:54:27:2c:a0:11:5d: - 55:b8:fe:01:71:66:4e:e4:53:c4:f0:79:95:12:e8: - b0:18:3d:f6:09:9e:57:ed:79:be:b3:df:b2:c0:6e: - da:03:bf:32:94:28:11:03:7f:f9:e6:a1:ae:c2:b8: - ac:52:3c:2e:47:20:a8:4b:6e:35:3f:ea:4b:8f:82: - 54:95:11:f2:4f:25:6b:b6:6d:92:43:02:41:4b:f3: - 93:ad:de:03:ec:d7:41:17:da:53:14:cb:42:6d:de: - d7:fe:4d:a4:23:12:f1:59:86:92:8d:f0:a0:1d:62: - 3d:de:56:03:9f:cc:80:5b:43:28:e8:b4:72:63:15: - ef:49:92:ae:2a:ed:1a:08:fa:77 + 00:bf:b8:22:3f:e1:e5:52:08:8a:f1:ff:f1:9c:08: + 2c:5e:40:fd:7a:32:d7:f4:c0:0b:88:34:14:7d:b9: + 3c:92:17:cb:04:56:84:ba:ac:d0:70:2d:eb:9d:65: + 76:87:99:ca:77:8c:ce:cd:d9:ef:46:8c:8f:1d:75: + 07:1c:3c:0a:4a:07:08:c0:c6:17:53:e1:77:ae:51: + 37:a9:e1:5a:da:29:68:cb:4d:60:ef:00:11:df:09: + 51:41:d6:d5:f7:1e:eb:04:59:a4:bc:9f:fa:47:f8: + de:0b:51:f9:f1:4b:71:e7:16:a1:82:e6:26:f6:0d: + 28:05:4c:29:cb:41:28:fc:da:25:20:00:27:5f:f3: + b8:99:4e:89:ec:fa:44:6f:a9:8b:1d:c2:9b:20:05: + 49:ce:fb:0f:a8:73:9a:3b:36:3d:5b:24:99:53:44: + 4b:18:ea:44:06:63:de:1a:b3:8a:42:92:6a:a1:a0: + 14:c2:ef:f9:5e:f7:01:46:99:78:a9:45:88:b7:a5: + f4:93:3a:a0:10:25:fa:96:d5:59:50:e0:16:dc:de: + 53:79:2f:e3:8d:4e:61:57:3a:04:c9:e6:a8:5a:4b: + c7:20:96:76:04:94:92:35:41:04:da:83:25:90:30: + 7b:81:0c:83:29:af:72:2f:ff:9a:94:53:37:48:33: + 3a:0c:ae:95:fa:89:e9:f7:19:a9:ff:8f:28:e3:ce: + f3:fd:e6:32:6f:1a:e8:d0:b9:dc:3f:9b:25:ca:f1: + 53:94:4e:15:6d:b2:d4:43:14:0d:dc:22:85:78:52: + a3:0e:0f:3f:30:ea:a6:14:d0:63:d8:14:1a:bf:ff: + 55:1d:a4:3a:e4:fd:f0:4f:05:ed:cc:2c:94:f6:fa: + c0:7f:91:3c:f0:c7:16:2c:2a:a7:1d:8c:f1:b6:06: + dd:d8:da:79:3b:84:2c:c4:d0:ab:ae:7a:a0:90:3e: + 4f:f7:7e:89:98:d5:30:f2:4e:55:8e:ae:20:12:e3: + 41:18:b9:ee:51:87:1f:6a:af:c9 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Alternative Name: @@ -90,9 +90,9 @@ Certificate: X509v3 Basic Constraints: critical CA:FALSE X509v3 Subject Key Identifier: - 1C:70:14:B0:20:DD:08:76:A4:3B:56:59:FA:5F:34:F8:36:66:E8:56 + E4:3D:4C:DB:96:85:CC:90:ED:0F:58:1E:8D:4D:7D:34:6C:4E:11:10 X509v3 Authority Key Identifier: - keyid:F3:EC:94:8E:F2:8E:30:C4:8E:68:C2:BF:8E:6A:19:C0:C1:9F:76:65 + keyid:C0:0A:2B:28:43:DE:5F:C9:7D:47:E5:47:9B:36:F2:65:8C:67:3B:E2 DirName:/C=XY/O=Python Software Foundation CA/CN=our-ca-server serial:CB:2D:80:99:5A:69:52:5B Authority Information Access: @@ -103,59 +103,59 @@ Certificate: URI:http://testca.pythontest.net/testca/revocation.crl Signature Algorithm: sha256WithRSAEncryption Signature Value: - 75:14:e5:68:45:8d:ed:6c:f1:27:1e:0e:f3:35:ae:0e:60:c1: - 65:36:62:b8:07:78:e1:b9:8d:7a:50:70:af:06:c9:d4:ee:50: - ef:d2:76:b2:a2:b6:cb:dc:a6:18:b5:3d:d2:f7:eb:0e:ec:b7: - 95:cd:2e:b1:36:6f:a8:9f:b8:4d:ff:ce:8a:c4:8e:62:37:32: - 80:3e:05:4a:4d:39:87:69:09:00:e8:40:64:d2:9d:f9:1f:9f: - ab:67:1f:f9:c6:84:ba:7e:17:6c:8b:8d:08:ee:fb:8a:d7:cd: - 06:25:72:9f:4e:1a:c2:71:e1:1b:cf:a2:d7:1c:05:12:95:d6: - 49:4b:e9:95:95:89:cf:68:18:46:a3:ea:0d:9d:8e:ca:1c:28: - 55:49:6b:c0:4b:58:f5:42:b9:0a:ec:0e:6e:21:a4:ff:60:c0: - 1b:6e:40:72:d0:a5:c5:b5:db:4e:87:67:3a:31:70:cb:32:84: - 70:a9:e2:ff:e0:f2:db:cd:03:b4:85:45:d3:07:cc:0f:c7:49: - d8:c2:17:eb:73:f7:4a:c0:d9:8c:59:ef:c0:0a:ce:13:0b:84: - c9:aa:0d:11:14:b4:e5:74:aa:ec:18:de:5f:26:18:98:4a:76: - f0:7f:cd:e6:c4:b5:58:03:03:f5:10:01:5d:8f:63:88:ba:65: - d7:b4:7f:5a:1a:51:0e:ed:e5:68:fa:18:03:72:15:a1:ec:27: - 1f:ea:ac:24:46:18:6e:f1:97:db:4a:f4:d6:a1:91:a0:8c:b0: - 2f:be:87:3b:44:b0:8d:2a:89:85:5f:f2:d9:e3:2e:66:b2:88: - 98:04:2c:96:32:38:99:19:a9:83:fd:94:0c:dd:63:d4:1b:60: - 9d:43:98:35:ac:b4:23:38:de:7f:85:52:57:a0:37:df:a5:cf: - be:54:2c:3c:50:27:2b:d4:54:a9:9d:a3:d4:a5:b3:c0:ea:3d: - 0e:e2:70:6b:fb:cb:a5:56:05:ec:64:72:f0:1a:db:64:01:cb: - 5d:27:c4:a1:c4:63 + 63:e9:cd:0f:ec:dd:27:a0:fc:76:16:a1:f2:5f:d3:07:9a:3f: + 97:f4:11:e8:20:e9:3c:dd:16:4f:20:62:71:73:a9:46:3e:85: + 6e:44:be:9d:08:4b:eb:80:08:d5:1b:93:5f:4a:0f:df:b7:1b: + 38:e2:2b:5c:68:48:ea:0a:58:5c:36:6c:79:a4:1e:b5:7e:ef: + cb:8d:5d:bd:7d:4a:e6:4e:dd:0b:87:10:ff:01:0e:9b:8b:bd: + de:1c:9a:25:fb:a2:e1:52:7b:8a:aa:08:37:b2:87:f7:45:9d: + 0b:ab:11:6b:0c:7f:db:ed:de:cc:1e:86:f0:be:30:25:6b:4b: + ff:f5:f2:99:ed:b0:b1:68:c1:44:0b:79:7b:81:95:b5:36:37: + 12:c4:99:9e:85:e8:2f:2d:cc:cd:d8:c6:f2:20:ec:6a:06:fd: + b8:fc:ff:02:11:95:4d:38:0d:42:8b:b2:43:eb:c2:b9:a4:e0: + 33:f1:da:25:2f:11:cf:57:b9:25:e5:ab:92:f2:b2:b5:11:2f: + bd:31:f7:55:eb:96:94:78:5a:ad:9c:8f:56:ec:34:a0:73:9c: + 01:c2:76:24:e0:f8:b3:c9:23:b9:ea:ab:c3:0a:d3:f8:53:44: + c7:12:37:76:44:d7:ee:25:93:f4:1c:1d:7c:fe:06:2c:fa:c5: + bb:c6:90:a4:57:fd:09:2e:da:af:f3:ac:e6:6d:7f:03:a1:26: + 36:9a:51:7d:8d:28:a7:5d:5d:37:57:cc:6a:11:2a:98:1d:57: + 71:32:6e:98:7c:12:28:ef:5c:2f:26:29:d4:56:0a:b1:23:d9: + 9d:35:20:1a:ec:cc:51:53:6e:ef:1c:e7:bc:3c:21:e7:64:31: + b1:4c:f3:55:b5:c4:93:17:d8:72:5a:05:2e:4a:e5:33:07:0e: + 7d:cf:22:73:0b:67:9f:b4:82:60:cd:71:f8:76:0c:c4:dc:98: + ee:49:f9:03:7f:0d:d8:c6:76:79:3c:28:ed:77:77:d3:7e:d5: + aa:1e:6e:2e:df:a5 -----BEGIN CERTIFICATE----- -MIIF9zCCBF+gAwIBAgIJAMstgJlaaVJdMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +MIIF+TCCBGGgAwIBAgIJAMstgJlaaVJdMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW -MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODA4MjkxNDIzMTZaFw0zNzEwMjgx -NDIzMTZaMGIxCzAJBgNVBAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEj -MCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xFTATBgNVBAMMDGZh -a2Vob3N0bmFtZTCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAMYoDXNk -ON40+HYn1YdVdcxR/yHk3j9scIzcJshpnV1KGQKB+YVaY62u4SKGBt9FhpJZkDAK -ZUBAaOaC9R+A4iCwV8zzI77MMHI+HNrte9Y9l8R7uY1EV7lmVie1t4z++5YIzhXP -AOr+aoVMXE1GbAieoTuHHRad8KhBZ703jWEnKXUrQ6MANv5KOWknT9QnreGwH52j -AaNpoG8nwd25BxGo2DyzoVzIknwyOyzW5EwUraP1CuaBsDrVe4g8LHNzcnnuZbwU -3YqxHwPYc6F72exMaXGn47kK51/9EFYHjjfFyjJ27BFc4VN0QBTjkxLi0oY/aHN7 -O7EwVCcsoBFdVbj+AXFmTuRTxPB5lRLosBg99gmeV+15vrPfssBu2gO/MpQoEQN/ -+eahrsK4rFI8LkcgqEtuNT/qS4+CVJUR8k8la7ZtkkMCQUvzk63eA+zXQRfaUxTL -Qm3e1/5NpCMS8VmGko3woB1iPd5WA5/MgFtDKOi0cmMV70mSrirtGgj6dwIDAQAB -o4IBwzCCAb8wFwYDVR0RBBAwDoIMZmFrZWhvc3RuYW1lMA4GA1UdDwEB/wQEAwIF -oDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAd -BgNVHQ4EFgQUHHAUsCDdCHakO1ZZ+l80+DZm6FYwfQYDVR0jBHYwdIAU8+yUjvKO -MMSOaMK/jmoZwMGfdmWhUaRPME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRo -b24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZl -coIJAMstgJlaaVJbMIGDBggrBgEFBQcBAQR3MHUwPAYIKwYBBQUHMAKGMGh0dHA6 -Ly90ZXN0Y2EucHl0aG9udGVzdC5uZXQvdGVzdGNhL3B5Y2FjZXJ0LmNlcjA1Bggr -BgEFBQcwAYYpaHR0cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0Y2Evb2Nz -cC8wQwYDVR0fBDwwOjA4oDagNIYyaHR0cDovL3Rlc3RjYS5weXRob250ZXN0Lm5l -dC90ZXN0Y2EvcmV2b2NhdGlvbi5jcmwwDQYJKoZIhvcNAQELBQADggGBAHUU5WhF -je1s8SceDvM1rg5gwWU2YrgHeOG5jXpQcK8GydTuUO/SdrKitsvcphi1PdL36w7s -t5XNLrE2b6ifuE3/zorEjmI3MoA+BUpNOYdpCQDoQGTSnfkfn6tnH/nGhLp+F2yL -jQju+4rXzQYlcp9OGsJx4RvPotccBRKV1klL6ZWVic9oGEaj6g2djsocKFVJa8BL -WPVCuQrsDm4hpP9gwBtuQHLQpcW1206HZzoxcMsyhHCp4v/g8tvNA7SFRdMHzA/H -SdjCF+tz90rA2YxZ78AKzhMLhMmqDREUtOV0quwY3l8mGJhKdvB/zebEtVgDA/UQ -AV2PY4i6Zde0f1oaUQ7t5Wj6GANyFaHsJx/qrCRGGG7xl9tK9NahkaCMsC++hztE -sI0qiYVf8tnjLmayiJgELJYyOJkZqYP9lAzdY9QbYJ1DmDWstCM43n+FUlegN9+l -z75ULDxQJyvUVKmdo9Sls8DqPQ7icGv7y6VWBexkcvAa22QBy10nxKHEYw== +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAgFw0xODA4MjkxNDIzMTZaGA8yNTI1MTAy +ODE0MjMxNlowYjELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4 +MSMwIQYDVQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEVMBMGA1UEAwwM +ZmFrZWhvc3RuYW1lMIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEAv7gi +P+HlUgiK8f/xnAgsXkD9ejLX9MALiDQUfbk8khfLBFaEuqzQcC3rnWV2h5nKd4zO +zdnvRoyPHXUHHDwKSgcIwMYXU+F3rlE3qeFa2iloy01g7wAR3wlRQdbV9x7rBFmk +vJ/6R/jeC1H58Utx5xahguYm9g0oBUwpy0Eo/NolIAAnX/O4mU6J7PpEb6mLHcKb +IAVJzvsPqHOaOzY9WySZU0RLGOpEBmPeGrOKQpJqoaAUwu/5XvcBRpl4qUWIt6X0 +kzqgECX6ltVZUOAW3N5TeS/jjU5hVzoEyeaoWkvHIJZ2BJSSNUEE2oMlkDB7gQyD +Ka9yL/+alFM3SDM6DK6V+onp9xmp/48o487z/eYybxro0LncP5slyvFTlE4VbbLU +QxQN3CKFeFKjDg8/MOqmFNBj2BQav/9VHaQ65P3wTwXtzCyU9vrAf5E88McWLCqn +HYzxtgbd2Np5O4QsxNCrrnqgkD5P936JmNUw8k5Vjq4gEuNBGLnuUYcfaq/JAgMB +AAGjggHDMIIBvzAXBgNVHREEEDAOggxmYWtlaG9zdG5hbWUwDgYDVR0PAQH/BAQD +AgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMBAf8EAjAA +MB0GA1UdDgQWBBTkPUzbloXMkO0PWB6NTX00bE4REDB9BgNVHSMEdjB0gBTACiso +Q95fyX1H5UebNvJljGc74qFRpE8wTTELMAkGA1UEBhMCWFkxJjAkBgNVBAoMHVB5 +dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uIENBMRYwFAYDVQQDDA1vdXItY2Etc2Vy +dmVyggkAyy2AmVppUlswgYMGCCsGAQUFBwEBBHcwdTA8BggrBgEFBQcwAoYwaHR0 +cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0Y2EvcHljYWNlcnQuY2VyMDUG +CCsGAQUFBzABhilodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rlc3RjYS9v +Y3NwLzBDBgNVHR8EPDA6MDigNqA0hjJodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3Qu +bmV0L3Rlc3RjYS9yZXZvY2F0aW9uLmNybDANBgkqhkiG9w0BAQsFAAOCAYEAY+nN +D+zdJ6D8dhah8l/TB5o/l/QR6CDpPN0WTyBicXOpRj6FbkS+nQhL64AI1RuTX0oP +37cbOOIrXGhI6gpYXDZseaQetX7vy41dvX1K5k7dC4cQ/wEOm4u93hyaJfui4VJ7 +iqoIN7KH90WdC6sRawx/2+3ezB6G8L4wJWtL//Xyme2wsWjBRAt5e4GVtTY3EsSZ +noXoLy3MzdjG8iDsagb9uPz/AhGVTTgNQouyQ+vCuaTgM/HaJS8Rz1e5JeWrkvKy +tREvvTH3VeuWlHharZyPVuw0oHOcAcJ2JOD4s8kjueqrwwrT+FNExxI3dkTX7iWT +9BwdfP4GLPrFu8aQpFf9CS7ar/Os5m1/A6EmNppRfY0op11dN1fMahEqmB1XcTJu +mHwSKO9cLyYp1FYKsSPZnTUgGuzMUVNu7xznvDwh52QxsUzzVbXEkxfYcloFLkrl +MwcOfc8icwtnn7SCYM1x+HYMxNyY7kn5A38N2MZ2eTwo7Xd3037Vqh5uLt+l -----END CERTIFICATE----- diff --git a/Lib/test/certdata/keycertecc.pem b/Lib/test/certdata/keycertecc.pem index bd109921898..d503d49dc85 100644 --- a/Lib/test/certdata/keycertecc.pem +++ b/Lib/test/certdata/keycertecc.pem @@ -1,8 +1,8 @@ -----BEGIN PRIVATE KEY----- -MIG2AgEAMBAGByqGSM49AgEGBSuBBAAiBIGeMIGbAgEBBDDRUbCeT3hMph4Y/ahL -1sy9Qfy4DYotuAP06UetzG6syv+EoQ02kX3xvazqwiJDrEyhZANiAAQef97STEPn -4Nk6C153VEx24MNkJUcmLe771u6lr3Q8Em3J/YPaA1i9Ys7KZA3WvoKBPoWaaikn -4yLQbd/6YE6AAjMuaThlR1/cqH5QnmS3DXHUjmxnLjWy/dZl0CJG1qo= +MIG2AgEAMBAGByqGSM49AgEGBSuBBAAiBIGeMIGbAgEBBDBuhPMBtySJ8hFhn5hU +1yvdDujmB8ajjXNlkQ9GhGdghQIAfKmP9vWtSY+1+BRjpAahZANiAAQKLvQoetPb +S5tbgExB5vSDUtgc7hK1j6AN++En6AkW8KkK+O/31jkItqBoSQz8ts+VCU5cloTf +H6i5AS9zUkHr4DEyn+XxfiutzcNQPdQIBmygWAsUpa1XlZf8ExMFf/A= -----END PRIVATE KEY----- Certificate: Data: @@ -13,19 +13,19 @@ Certificate: Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server Validity Not Before: Aug 29 14:23:16 2018 GMT - Not After : Oct 28 14:23:16 2037 GMT + Not After : Oct 28 14:23:16 2525 GMT Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=localhost-ecc Subject Public Key Info: Public Key Algorithm: id-ecPublicKey Public-Key: (384 bit) pub: - 04:1e:7f:de:d2:4c:43:e7:e0:d9:3a:0b:5e:77:54: - 4c:76:e0:c3:64:25:47:26:2d:ee:fb:d6:ee:a5:af: - 74:3c:12:6d:c9:fd:83:da:03:58:bd:62:ce:ca:64: - 0d:d6:be:82:81:3e:85:9a:6a:29:27:e3:22:d0:6d: - df:fa:60:4e:80:02:33:2e:69:38:65:47:5f:dc:a8: - 7e:50:9e:64:b7:0d:71:d4:8e:6c:67:2e:35:b2:fd: - d6:65:d0:22:46:d6:aa + 04:0a:2e:f4:28:7a:d3:db:4b:9b:5b:80:4c:41:e6: + f4:83:52:d8:1c:ee:12:b5:8f:a0:0d:fb:e1:27:e8: + 09:16:f0:a9:0a:f8:ef:f7:d6:39:08:b6:a0:68:49: + 0c:fc:b6:cf:95:09:4e:5c:96:84:df:1f:a8:b9:01: + 2f:73:52:41:eb:e0:31:32:9f:e5:f1:7e:2b:ad:cd: + c3:50:3d:d4:08:06:6c:a0:58:0b:14:a5:ad:57:95: + 97:fc:13:13:05:7f:f0 ASN1 OID: secp384r1 NIST CURVE: P-384 X509v3 extensions: @@ -38,9 +38,9 @@ Certificate: X509v3 Basic Constraints: critical CA:FALSE X509v3 Subject Key Identifier: - 45:ED:32:14:6D:51:A2:3B:B0:80:55:E0:A6:9B:74:4C:A5:56:88:B1 + F3:39:E5:90:3F:58:6F:55:9D:91:D4:86:B8:1B:14:35:09:AC:06:97 X509v3 Authority Key Identifier: - keyid:F3:EC:94:8E:F2:8E:30:C4:8E:68:C2:BF:8E:6A:19:C0:C1:9F:76:65 + keyid:C0:0A:2B:28:43:DE:5F:C9:7D:47:E5:47:9B:36:F2:65:8C:67:3B:E2 DirName:/C=XY/O=Python Software Foundation CA/CN=our-ca-server serial:CB:2D:80:99:5A:69:52:5B Authority Information Access: @@ -51,53 +51,53 @@ Certificate: URI:http://testca.pythontest.net/testca/revocation.crl Signature Algorithm: sha256WithRSAEncryption Signature Value: - 07:e4:91:0b:d3:ed:4b:52:7f:50:68:c7:8d:80:48:9f:b7:4a: - 13:66:bf:9d:4c:2d:18:19:68:a0:da:3b:12:85:05:16:fa:8d: - 9c:58:c6:81:b3:96:ba:11:62:65:d3:76:f1:1c:ab:95:e4:d8: - 2a:e0:1f:7b:c5:20:2e:7c:8f:de:87:7a:2b:52:54:ca:d1:41: - b0:5e:20:72:df:44:00:4a:69:1a:ef:10:63:52:13:ed:49:02: - ee:dc:9d:f3:c8:ba:c4:01:81:5a:a9:1c:15:12:b6:21:de:44: - a5:fd:7e:f9:22:d1:3e:ee:22:dd:31:55:32:4e:41:68:27:c5: - 95:1b:7e:6b:18:74:f9:22:d6:b7:b9:31:72:51:a0:5a:2c:ff: - 62:76:e9:a0:55:8d:78:33:52:4a:58:b2:f4:4b:0c:43:82:2f: - a9:84:68:05:dd:11:47:70:24:fe:5c:92:fd:17:21:63:bb:fa: - 93:fa:54:54:05:72:48:ed:81:48:ab:95:fc:6d:a8:62:96:f9: - 3b:e2:71:18:05:3e:76:bb:df:95:17:7b:81:4b:1f:7f:e1:67: - 76:c4:07:cb:65:a7:f2:cf:e6:b4:fb:75:7c:ee:df:a1:f5:34: - 20:2b:48:fd:2e:49:ff:f3:a6:3b:00:49:6c:88:79:ed:9c:16: - 2a:04:72:e2:93:e4:7e:3f:2a:dd:30:47:9a:99:84:2a:b9:c4: - 40:31:a6:68:f3:20:d1:75:f1:1e:c8:18:64:5b:f8:4c:ce:9a: - 3c:57:2c:e3:63:64:29:0a:c2:b6:8e:20:01:55:9f:fe:10:ba: - 12:42:38:0a:9b:53:01:a5:b4:08:76:ec:e8:a6:fc:69:2c:f7: - 7f:5e:0f:44:07:55:e1:7c:2e:58:e5:d6:fc:6f:c2:4d:83:65: - bd:f3:32:e3:14:48:22:8d:80:18:ea:44:f8:24:79:ff:ff:c6: - 04:c2:e9:90:34:40:d6:59:3f:59:1e:4a:9a:58:60:ce:ab:f9: - 76:0e:ef:f7:05:17 + b8:d9:bf:b8:88:0a:01:2b:aa:64:32:e8:97:e1:0f:ee:34:40: + ef:71:fc:e5:f4:a2:3b:26:00:e2:19:3b:3e:cb:8e:2b:51:4c: + 30:ff:ab:44:9d:28:8d:d2:9c:32:e3:6b:96:73:1c:9d:55:76: + b2:bf:af:b8:51:ed:fd:04:e7:0d:fa:8c:2a:68:01:cb:92:90: + 7d:d1:31:d8:6c:f4:b8:4b:ea:62:59:1e:31:4e:f7:17:ae:3f: + f0:b6:ff:f8:6e:64:e3:6e:e9:19:4f:d0:1e:84:1e:df:56:49: + ae:90:a9:e0:7b:70:e6:97:7f:08:f2:03:49:82:7d:58:8e:a5: + a0:88:59:f3:a1:ab:b7:dd:6b:9a:2b:79:64:9c:d3:07:7c:ec: + a3:8d:56:77:28:10:c9:42:a5:fa:81:8e:35:ad:f7:28:da:58: + cc:a0:61:fa:0d:75:fd:26:ab:07:88:c5:64:21:f1:98:1c:e3: + 13:51:38:a7:59:a5:59:88:b9:10:4d:c1:79:70:3f:36:9e:08: + 75:53:7f:77:a3:3d:5d:16:b7:3b:a2:f6:eb:41:9b:56:16:80: + 34:96:57:f0:3a:3d:91:43:17:51:2b:64:2c:d7:e2:25:45:85: + f3:5f:73:f8:7d:aa:a3:6d:61:e8:ba:c5:90:5e:16:50:bc:79: + b9:4e:df:66:db:ae:95:80:15:da:4e:bf:8f:4e:9d:e3:7d:b0: + ab:8b:72:93:3c:56:1b:92:d3:67:07:d5:5a:ad:98:16:69:ea: + 46:fe:e0:d0:f4:ae:95:d3:b5:80:7d:f2:64:4d:14:c0:97:d2: + f3:91:bb:e6:43:d0:7e:39:14:0b:95:ab:c1:56:fd:26:79:f8: + 8c:69:eb:bc:74:0e:ea:cd:94:62:7e:30:58:01:7f:84:ee:9d: + 58:9c:fc:8f:75:1e:d9:86:cb:f4:fd:4c:af:b2:f7:69:ea:58: + cb:7c:93:ce:3d:86:f0:46:01:df:86:02:a1:6c:0f:fb:13:84: + d6:75:23:f9:17:21 -----BEGIN CERTIFICATE----- -MIIEyzCCAzOgAwIBAgIJAMstgJlaaVJeMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +MIIEzTCCAzWgAwIBAgIJAMstgJlaaVJeMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW -MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODA4MjkxNDIzMTZaFw0zNzEwMjgx -NDIzMTZaMGMxCzAJBgNVBAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEj -MCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xFjAUBgNVBAMMDWxv -Y2FsaG9zdC1lY2MwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAAQef97STEPn4Nk6C153 -VEx24MNkJUcmLe771u6lr3Q8Em3J/YPaA1i9Ys7KZA3WvoKBPoWaaikn4yLQbd/6 -YE6AAjMuaThlR1/cqH5QnmS3DXHUjmxnLjWy/dZl0CJG1qqjggHEMIIBwDAYBgNV -HREEETAPgg1sb2NhbGhvc3QtZWNjMA4GA1UdDwEB/wQEAwIFoDAdBgNVHSUEFjAU -BggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQURe0y -FG1RojuwgFXgppt0TKVWiLEwfQYDVR0jBHYwdIAU8+yUjvKOMMSOaMK/jmoZwMGf -dmWhUaRPME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUg -Rm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcoIJAMstgJlaaVJb -MIGDBggrBgEFBQcBAQR3MHUwPAYIKwYBBQUHMAKGMGh0dHA6Ly90ZXN0Y2EucHl0 -aG9udGVzdC5uZXQvdGVzdGNhL3B5Y2FjZXJ0LmNlcjA1BggrBgEFBQcwAYYpaHR0 -cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0Y2Evb2NzcC8wQwYDVR0fBDww -OjA4oDagNIYyaHR0cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0Y2EvcmV2 -b2NhdGlvbi5jcmwwDQYJKoZIhvcNAQELBQADggGBAAfkkQvT7UtSf1Box42ASJ+3 -ShNmv51MLRgZaKDaOxKFBRb6jZxYxoGzlroRYmXTdvEcq5Xk2CrgH3vFIC58j96H -eitSVMrRQbBeIHLfRABKaRrvEGNSE+1JAu7cnfPIusQBgVqpHBUStiHeRKX9fvki -0T7uIt0xVTJOQWgnxZUbfmsYdPki1re5MXJRoFos/2J26aBVjXgzUkpYsvRLDEOC -L6mEaAXdEUdwJP5ckv0XIWO7+pP6VFQFckjtgUirlfxtqGKW+TvicRgFPna735UX -e4FLH3/hZ3bEB8tlp/LP5rT7dXzu36H1NCArSP0uSf/zpjsASWyIee2cFioEcuKT -5H4/Kt0wR5qZhCq5xEAxpmjzINF18R7IGGRb+EzOmjxXLONjZCkKwraOIAFVn/4Q -uhJCOAqbUwGltAh27Oim/Gks939eD0QHVeF8Lljl1vxvwk2DZb3zMuMUSCKNgBjq -RPgkef//xgTC6ZA0QNZZP1keSppYYM6r+XYO7/cFFw== +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAgFw0xODA4MjkxNDIzMTZaGA8yNTI1MTAy +ODE0MjMxNlowYzELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4 +MSMwIQYDVQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEWMBQGA1UEAwwN +bG9jYWxob3N0LWVjYzB2MBAGByqGSM49AgEGBSuBBAAiA2IABAou9Ch609tLm1uA +TEHm9INS2BzuErWPoA374SfoCRbwqQr47/fWOQi2oGhJDPy2z5UJTlyWhN8fqLkB +L3NSQevgMTKf5fF+K63Nw1A91AgGbKBYCxSlrVeVl/wTEwV/8KOCAcQwggHAMBgG +A1UdEQQRMA+CDWxvY2FsaG9zdC1lY2MwDgYDVR0PAQH/BAQDAgWgMB0GA1UdJQQW +MBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMBAf8EAjAAMB0GA1UdDgQWBBTz +OeWQP1hvVZ2R1Ia4GxQ1CawGlzB9BgNVHSMEdjB0gBTACisoQ95fyX1H5UebNvJl +jGc74qFRpE8wTTELMAkGA1UEBhMCWFkxJjAkBgNVBAoMHVB5dGhvbiBTb2Z0d2Fy +ZSBGb3VuZGF0aW9uIENBMRYwFAYDVQQDDA1vdXItY2Etc2VydmVyggkAyy2AmVpp +UlswgYMGCCsGAQUFBwEBBHcwdTA8BggrBgEFBQcwAoYwaHR0cDovL3Rlc3RjYS5w +eXRob250ZXN0Lm5ldC90ZXN0Y2EvcHljYWNlcnQuY2VyMDUGCCsGAQUFBzABhilo +dHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rlc3RjYS9vY3NwLzBDBgNVHR8E +PDA6MDigNqA0hjJodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rlc3RjYS9y +ZXZvY2F0aW9uLmNybDANBgkqhkiG9w0BAQsFAAOCAYEAuNm/uIgKASuqZDLol+EP +7jRA73H85fSiOyYA4hk7PsuOK1FMMP+rRJ0ojdKcMuNrlnMcnVV2sr+vuFHt/QTn +DfqMKmgBy5KQfdEx2Gz0uEvqYlkeMU73F64/8Lb/+G5k427pGU/QHoQe31ZJrpCp +4Htw5pd/CPIDSYJ9WI6loIhZ86Grt91rmit5ZJzTB3zso41WdygQyUKl+oGONa33 +KNpYzKBh+g11/SarB4jFZCHxmBzjE1E4p1mlWYi5EE3BeXA/Np4IdVN/d6M9XRa3 +O6L260GbVhaANJZX8Do9kUMXUStkLNfiJUWF819z+H2qo21h6LrFkF4WULx5uU7f +ZtuulYAV2k6/j06d432wq4tykzxWG5LTZwfVWq2YFmnqRv7g0PSuldO1gH3yZE0U +wJfS85G75kPQfjkUC5WrwVb9Jnn4jGnrvHQO6s2UYn4wWAF/hO6dWJz8j3Ue2YbL +9P1Mr7L3aepYy3yTzj2G8EYB34YCoWwP+xOE1nUj+Rch -----END CERTIFICATE----- diff --git a/Lib/test/certdata/make_ssl_certs.py b/Lib/test/certdata/make_ssl_certs.py index ed2037c1fdf..18e61449638 100644 --- a/Lib/test/certdata/make_ssl_certs.py +++ b/Lib/test/certdata/make_ssl_certs.py @@ -1,6 +1,7 @@ """Make the custom certificate and private key files used by test_ssl and friends.""" +import argparse import os import pprint import shutil @@ -8,7 +9,8 @@ from subprocess import * startdate = "20180829142316Z" -enddate = "20371028142316Z" +enddate_default = "25251028142316Z" +days_default = "140000" req_template = """ [ default ] @@ -79,8 +81,8 @@ default_startdate = {startdate} enddate = {enddate} default_enddate = {enddate} - default_days = 7000 - default_crl_days = 7000 + default_days = {days} + default_crl_days = {days} certificate = pycacert.pem private_key = pycakey.pem serial = $dir/serial @@ -117,7 +119,7 @@ here = os.path.abspath(os.path.dirname(__file__)) -def make_cert_key(hostname, sign=False, extra_san='', +def make_cert_key(cmdlineargs, hostname, sign=False, extra_san='', ext='req_x509_extensions_full', key='rsa:3072'): print("creating cert for " + hostname) tempnames = [] @@ -130,13 +132,13 @@ def make_cert_key(hostname, sign=False, extra_san='', hostname=hostname, extra_san=extra_san, startdate=startdate, - enddate=enddate + enddate=cmdlineargs.enddate, + days=cmdlineargs.days ) with open(req_file, 'w') as f: f.write(req) - args = ['req', '-new', '-nodes', '-days', '7000', + args = ['req', '-new', '-nodes', '-days', cmdlineargs.days, '-newkey', key, '-keyout', key_file, - '-extensions', ext, '-config', req_file] if sign: with tempfile.NamedTemporaryFile(delete=False) as f: @@ -145,7 +147,7 @@ def make_cert_key(hostname, sign=False, extra_san='', args += ['-out', reqfile ] else: - args += ['-x509', '-out', cert_file ] + args += ['-extensions', ext, '-x509', '-out', cert_file ] check_call(['openssl'] + args) if sign: @@ -175,7 +177,7 @@ def make_cert_key(hostname, sign=False, extra_san='', def unmake_ca(): shutil.rmtree(TMP_CADIR) -def make_ca(): +def make_ca(cmdlineargs): os.mkdir(TMP_CADIR) with open(os.path.join('cadir','index.txt'),'a+') as f: pass # empty file @@ -192,7 +194,8 @@ def make_ca(): hostname='our-ca-server', extra_san='', startdate=startdate, - enddate=enddate + enddate=cmdlineargs.enddate, + days=cmdlineargs.days ) t.write(req) t.flush() @@ -219,14 +222,22 @@ def make_ca(): shutil.copy('capath/ceff1710.0', 'capath/b1930218.0') -def print_cert(path): +def write_cert_reference(path): import _ssl - pprint.pprint(_ssl._test_decode_cert(path)) + refdata = pprint.pformat(_ssl._test_decode_cert(path)) + print(refdata) + with open(path + '.reference', 'w') as f: + print(refdata, file=f) if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Make the custom certificate and private key files used by test_ssl and friends.') + parser.add_argument('--days', default=days_default) + parser.add_argument('--enddate', default=enddate_default) + cmdlineargs = parser.parse_args() + os.chdir(here) - cert, key = make_cert_key('localhost', ext='req_x509_extensions_simple') + cert, key = make_cert_key(cmdlineargs, 'localhost', ext='req_x509_extensions_simple') with open('ssl_cert.pem', 'w') as f: f.write(cert) with open('ssl_key.pem', 'w') as f: @@ -243,26 +254,26 @@ def print_cert(path): f.write(cert) # For certificate matching tests - make_ca() - cert, key = make_cert_key('fakehostname', ext='req_x509_extensions_simple') + make_ca(cmdlineargs) + cert, key = make_cert_key(cmdlineargs, 'fakehostname', ext='req_x509_extensions_simple') with open('keycert2.pem', 'w') as f: f.write(key) f.write(cert) - cert, key = make_cert_key('localhost', sign=True) + cert, key = make_cert_key(cmdlineargs, 'localhost', sign=True) with open('keycert3.pem', 'w') as f: f.write(key) f.write(cert) check_call(['openssl', 'x509', '-outform', 'pem', '-in', 'keycert3.pem', '-out', 'cert3.pem']) - cert, key = make_cert_key('fakehostname', sign=True) + cert, key = make_cert_key(cmdlineargs, 'fakehostname', sign=True) with open('keycert4.pem', 'w') as f: f.write(key) f.write(cert) cert, key = make_cert_key( - 'localhost-ecc', sign=True, key='param:secp384r1.pem' + cmdlineargs, 'localhost-ecc', sign=True, key='param:secp384r1.pem' ) with open('keycertecc.pem', 'w') as f: f.write(key) @@ -282,7 +293,7 @@ def print_cert(path): 'RID.1 = 1.2.3.4.5', ] - cert, key = make_cert_key('allsans', sign=True, extra_san='\n'.join(extra_san)) + cert, key = make_cert_key(cmdlineargs, 'allsans', sign=True, extra_san='\n'.join(extra_san)) with open('allsans.pem', 'w') as f: f.write(key) f.write(cert) @@ -299,17 +310,17 @@ def print_cert(path): ] # IDN SANS, signed - cert, key = make_cert_key('idnsans', sign=True, extra_san='\n'.join(extra_san)) + cert, key = make_cert_key(cmdlineargs, 'idnsans', sign=True, extra_san='\n'.join(extra_san)) with open('idnsans.pem', 'w') as f: f.write(key) f.write(cert) - cert, key = make_cert_key('nosan', sign=True, ext='req_x509_extensions_nosan') + cert, key = make_cert_key(cmdlineargs, 'nosan', sign=True, ext='req_x509_extensions_nosan') with open('nosan.pem', 'w') as f: f.write(key) f.write(cert) unmake_ca() - print("update Lib/test/test_ssl.py and Lib/test/test_asyncio/utils.py") - print_cert('keycert.pem') - print_cert('keycert3.pem') + print("Writing out reference data for Lib/test/test_ssl.py and Lib/test/test_asyncio/utils.py") + write_cert_reference('keycert.pem') + write_cert_reference('keycert3.pem') diff --git a/Lib/test/certdata/nosan.pem b/Lib/test/certdata/nosan.pem index c6ff8ea31bf..c397342fa86 100644 --- a/Lib/test/certdata/nosan.pem +++ b/Lib/test/certdata/nosan.pem @@ -1,131 +1,137 @@ -----BEGIN PRIVATE KEY----- -MIIG/gIBADANBgkqhkiG9w0BAQEFAASCBugwggbkAgEAAoIBgQC99xEYPTwFN/ji -i0lm11ckEGhcxciSsIgTgior54CLgQy7JXllTYmAWFTTg2zNBvDMexGI0h+xtZ4q -1Renghgt33N3Y6CT3v/L7JkE1abQbFveKW/ydlxH0+jLlsENSWjySwC80+f9L3bX -TcD8T4Fu9Uty2Rg1a/Eyekng5RmfkmLNgxfnX5R5nWhh0Aia7h3Ax2zCALfxqZIm -fxwavEgHsW/yZi+T+eoJwe0i7a6LaUoLqsPV9ZhagziNDaappPHH42NW39WlRhx1 -UjtiRm2Jihnzxcfs+90zLXSp5pxo/cE9Ia4d8ieq3Rxd/XgjlF6FXXFJjwfL36Dw -ehy8m3PKKAuO+fyMgPPPMQb7oaRy/MBG0NayRreTwyKILS2zafIW/iKpgICbxrWJ -r/H1b3S6PBKYUE2uQs0/ZPnRjjh0VeNnue7JcRoNbe27I2d56KUBsVEPdokjU59v -NYi6Se+ViZXtUbM1u/I0kvDMprAiobwtJFYgcE86N1lFJjHSwDMCAwEAAQKCAYBb -lvnJBA0iPwBiyeFUElNTcg2/XST9hNu2/DU1AeM6X7gxqznCnAXFudD8Qgt9NvF2 -xYeIvjbFydk+sYs8Gj9qLqhPUdukMAqI2cRVTmWla/lHPhdZgbOwdf1x23es3k4Z -NAxg/pKFwhK8cCKyA+tWAjKkZwODDk42ljt0kUEvbLbye1hVGAJQOJKRRmo/uLrj -rcNELnCBtc5ffT2hrlHUU7qz1ozt/brXhYa+JnbXhKZMxcKyMD2KtmXXrFNEy99o -jXbrpDCos82bzQfPDo8IpCbVbEd2J00aFmrNjQWhZuXX5dXflrujW4J0nzeHrZ78 -rNAz2/YuZ543BTB3XbogeFuLC5RqBgAMmw2WJ96Oa/UG8nZNvEw54N5r6dhfXj6A -VlJFLVwlfBQdAdaM3P4uZ6WECrH3EerQa27qyUdRrcDaGPLt7wG9FmMivnW1KQsy -5ow/gM0CsxFj2xNoGw1S5jtclbgSy8HNJaBsNk4XMQ+ORABZdG1MTTE+GMSjD/EC -gcEA+6JYiZEo+QrvItIZYB6Go4suu/F8df1pEjJlxwp2GmObitRhtV6r9g9IySFv -5SL7ZxARr4aQxvM7fNp57p9ssmkBtY0ofMjJAxhvs4T37bAcGK/2xCegNSmbqh24 -FAWpRDMgE5PjtuWC5jTvSOYFeUxwI/cu0HxWdxJl2dPUSL1nI2jP+ok3pZobEQk9 -E2+MlHpKmU+s/lAkuQiP+AW9a4M+ZJNWxocJjmtwj4OjJXPm7GddA/5x622DxFe6 -4K2vAoHBAMFC0An25bwGoFrCV/96s45K4qZcZcJ660+aK3xXaq6/8KfiusJnWds2 -nc0B6jYjKs8A7yTAGXke6fmyVsoLosZiXsbpW2m16g8jL79Tc85O9oDNmDIGk1uT -tRLZc2BvmHmy/dNrdbT/EHC3FKNWQVqWc2sHhPeB6F3hIEXDSUO/GB0njMZNXrPJ -547RlhN0xCLb3vTzzGHwNmwfI81YjV/XI4vpJjq1YceN8Xyd1r5ZOFfU8woIACO3 -I4dvBQ1avQKBwQCLDs9wzohfAFzg2Exvos7y6AKemDgYmD8NcE5wbWaQ9MTLNsz8 -RuIu64lkpRbKAMf/z5CGeI3fdCFGwRGq/e06tu7b3rMmKmtzS3jHM08zyiPsvKlZ -AzD00BaXLy8/2VUOPFaYmxy3QSRShaRKm9sgik5agcocKuo5iTBB7V8eB5VMqyps -IJJg8MXOZ1WaPQXqM56wFKjcLXvtyT6OaNWh6Xh8ajQFKDDuxI8CsFNjaiaONBzi -DSX1XaL4ySab7T8CgcEAsI+7xP6+EDP1mDVpc8zD8kHUI6zSgwUNqiHtjKHIo3JU -CO2JNkZ5v158eGlBcshaOdheozKlkxR9KlSWGezbf2crs4pKq585AS9iVeeGK3vU -lQRAAaQkSEv/6AKl9/q8UKMIZnkMhploibGZt0f8WSiOtb+e6QjUI8CjXVj2vF// -RdN2N01EMflKBh7Qf2H0NuytGxkJJojxD4K7kMVQE7lXjmEpPgWsGUZC01jYcfrN -EOFKUWXRys9sNDVnZjX5AoHAFRyOC1BlmVEtcOsgzum4+JEDWvRnO1hP1tm68eTZ -ijB/XppDtVESpq3+1+gx2YOmzlUNEhKlcn6eHPWEJwdVxJ87Gdh03rIV/ZQUKe46 -3+j6l/5coN4WfCBloy4b+Tcj+ZTL4sKaLm33RoD2UEWS5mmItfZuoEFQB958h3JD -1Ka1tgsLnuYGjcrg+ALvbM5nQlefzPqPJh0C8UV3Ny/4Gd02YgHw7Yoe4m6OUqQv -hctFUL/gjIGg9PVqTWzVVKaI +MIIG/QIBADANBgkqhkiG9w0BAQEFAASCBucwggbjAgEAAoIBgQDD5QDHBBTqG5g/ +OW6kVb6oTHLM4q6wX42YofPGBjWFlIeMX78A1PzRvgKLo9RXbBOb/Q89rbtsL9dd ++FcUaNP7GNLnX+MpeOnMOL6M7LzihFi0OyRmvCytWIUMYkH8lqHuydwdUROsbqGk +2r7Sbe8XzDbf9dNnCYZnoOJMiGvp08D5FFGJL+K7N9xCEHCpHJ5mFyywzunoR3us +EgV+0b8nt287RcLoBNaORCLt6L1FSsSb6g3NS/AUNNDb1VLoa96HFLL3rBCqSt3y +SCGtz3lncEG5HXBRNEWoka4SKqHDOP8zDZZxtmNwWp6QtHaFj4rY/XnMGiekJgIG +crzeB8jAgwGC9u23Yo5/IX5OhgSIbqj0mtYVRwZwJbszXVQn7BNbQzNFNgnpJPpJ +xA4xDRr415IsAQHm7ugt3G0aX2PbbXYxOpF/Ud+3JQu5vYGbcMUxqOPaHi3C8QUT +rcvIRBm4W2L2fd3xANpC6RafRVXjfeGFfk0pyArlvIyYCGvxAeUCAwEAAQKCAYAE +A1FQwmMxHOhUbuAIRoUInwhMeLs7ZKJx8OsGPDSCTGRGGnAFD/rSP63cNy71T+AF +gggpiGQegz9KkeGfhQwM1GssLGk8WnCp2fnqbAwyhkZzpOs+WEJSCsKMFDc3S0GU +hChIYAJFtgRLJnAFM+b2V0uIfHCz26hGqKOdDb/Wumzd8zPhH+/SNI5hdM7xStez +2d/dlqVZ21c0ghiMLZ/M2qrz8EEvWccc4BEuYtqJYNpdgAauLIGF0SxByPt/oZMf +4fa0zOVfAL6F5cPFkycRzuz97Lx+ZdtLuUBx/3y8Sk24TR1QutwaFhbwpRSQSUrg +xT7ZmS24rQhaP8i7+ROLGy0CPlLMc/D3WvvtdLDWOUF30azURvAFcqKA87Bp05bR +HI/ewXaSB26J34MUmBHy5dBjld+HX6hQInPhCTZu7gNDOrPaEc8Ug50c1r7HkYRl +VBv/4BQo3YtNdoKhL08MVuNbcxrzge3kI0mbQu7qDSyrve02WnQ1zijJ916NbgEC +gcEA6oFNNtWzVw/7/MBxUZ1tE5VW5U2SjRvQZlcPTJ9Ub16byhOCdNvE17uyYLt5 +diqoVAvSGWiJ+OKsmw3XTLdhxzWeZ1a9ZiCozsmIfHMQmL9t3MxzTnMEZCfcvxz0 +DPvAN0REUtwRkMbt7vC+g01Eom9UpAH7hKXkc3pTgtkoyFeebC/YneGtdOTVATKU +WMfsY6Biye7P1D8V7btjLI8LTnv90ygoP0uSrX53hp7k4EFAeXyT9VN5wMXLw81a +5K69AoHBANXZsvSSIwe7fjspNUU9yELnEokDIdBn4ARX6x5PonL2KB0iVUzWgjHb +vs23GAzAgKKY+S8VLYptBLUBlsZ/pSLfZLWd1ywxeHmk/nkzuqlAw1jwuUlMq6Vr +BJmfhpOoXeSrQwvkL/uOejdrDSmLAFiIV3l9LtkmIJzfQSSy2QB8mfWwni0vxOJT +mIqZE01rfRTNLIqxO3+0d7pDqArVvWxYqF4C8kihUhNjkB5NaURUTzDQ3Y33aAvT +zAm697nGSQKBwQDEQ8ed9ykL2sLpfR7aUclytHBvpYbcNsUqgf66ADeopiP48m8i +4rRSYjMepok3jugmv2XuAgJHnV8cvm7NNEXPdl7G2l/V08u0lhN3JM5lKQIH4801 +gSnRsVMdWFwhaaosFySfvLOu2e9VJYQtXEPvNwI96bLaCAW1aFHwl1N8qWhb34eK +S9DinopvYCesTlbX4uoLW6XxW4M83rJYHrg1zaxYR6m3n8Z5EflzYBTqY3JUuyES +F/U0k9bAX2SNNHkCgcALk8mYbADxfjkLQuPbZ8jbtl7OhBjki3sZQRk9ftowlxr8 +2Mr9ae+Ke3cM9AidSB6urtFutxrMD7LdicR74pUyGh39pxnrDpKTI1eTgDVuzE7H +FeEyErCIOA77siM7AzZyFsN+dVATslbzgRwpT5kpMdhqf1h18RZ656tDLVuKJzS+ +lF073QYvqo7rkfX1jwgqhCERMR8jfsWsk9UZIREsOHCFBmvPesxSuGUo/s/gHyBa +aDRWZzp+yWyWakTXDeECgcBaPIxLK0ky89Mv8jTSyIJIMhQj8+z8wFLJFQG5vF/Z +1taLIDY0stU8HzKflTJ5v72r2jpIvu5YgDEgl+kKjyqLMWSu4nBZ2OoJet49S6d3 +X0YL2VzsIiSG+8cmVvBBf7iZCHDKiCM20dzzhZq8BCLiaPq+Zu2No2xEJyE0m8rc +I66z6fV1BWQXLe6vPC8EVvBWz3Ybje0czUdZnZ44qV6qyZzREXuqbC0qS5RanE4Z ++Ps9b3TezE7gDII6SbUwhZo= -----END PRIVATE KEY----- Certificate: Data: - Version: 1 (0x0) + Version: 3 (0x2) Serial Number: cb:2d:80:99:5a:69:52:61 Signature Algorithm: sha256WithRSAEncryption Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server Validity Not Before: Aug 29 14:23:16 2018 GMT - Not After : Oct 28 14:23:16 2037 GMT + Not After : Oct 28 14:23:16 2525 GMT Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=nosan Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (3072 bit) Modulus: - 00:bd:f7:11:18:3d:3c:05:37:f8:e2:8b:49:66:d7: - 57:24:10:68:5c:c5:c8:92:b0:88:13:82:2a:2b:e7: - 80:8b:81:0c:bb:25:79:65:4d:89:80:58:54:d3:83: - 6c:cd:06:f0:cc:7b:11:88:d2:1f:b1:b5:9e:2a:d5: - 17:a7:82:18:2d:df:73:77:63:a0:93:de:ff:cb:ec: - 99:04:d5:a6:d0:6c:5b:de:29:6f:f2:76:5c:47:d3: - e8:cb:96:c1:0d:49:68:f2:4b:00:bc:d3:e7:fd:2f: - 76:d7:4d:c0:fc:4f:81:6e:f5:4b:72:d9:18:35:6b: - f1:32:7a:49:e0:e5:19:9f:92:62:cd:83:17:e7:5f: - 94:79:9d:68:61:d0:08:9a:ee:1d:c0:c7:6c:c2:00: - b7:f1:a9:92:26:7f:1c:1a:bc:48:07:b1:6f:f2:66: - 2f:93:f9:ea:09:c1:ed:22:ed:ae:8b:69:4a:0b:aa: - c3:d5:f5:98:5a:83:38:8d:0d:a6:a9:a4:f1:c7:e3: - 63:56:df:d5:a5:46:1c:75:52:3b:62:46:6d:89:8a: - 19:f3:c5:c7:ec:fb:dd:33:2d:74:a9:e6:9c:68:fd: - c1:3d:21:ae:1d:f2:27:aa:dd:1c:5d:fd:78:23:94: - 5e:85:5d:71:49:8f:07:cb:df:a0:f0:7a:1c:bc:9b: - 73:ca:28:0b:8e:f9:fc:8c:80:f3:cf:31:06:fb:a1: - a4:72:fc:c0:46:d0:d6:b2:46:b7:93:c3:22:88:2d: - 2d:b3:69:f2:16:fe:22:a9:80:80:9b:c6:b5:89:af: - f1:f5:6f:74:ba:3c:12:98:50:4d:ae:42:cd:3f:64: - f9:d1:8e:38:74:55:e3:67:b9:ee:c9:71:1a:0d:6d: - ed:bb:23:67:79:e8:a5:01:b1:51:0f:76:89:23:53: - 9f:6f:35:88:ba:49:ef:95:89:95:ed:51:b3:35:bb: - f2:34:92:f0:cc:a6:b0:22:a1:bc:2d:24:56:20:70: - 4f:3a:37:59:45:26:31:d2:c0:33 + 00:c3:e5:00:c7:04:14:ea:1b:98:3f:39:6e:a4:55: + be:a8:4c:72:cc:e2:ae:b0:5f:8d:98:a1:f3:c6:06: + 35:85:94:87:8c:5f:bf:00:d4:fc:d1:be:02:8b:a3: + d4:57:6c:13:9b:fd:0f:3d:ad:bb:6c:2f:d7:5d:f8: + 57:14:68:d3:fb:18:d2:e7:5f:e3:29:78:e9:cc:38: + be:8c:ec:bc:e2:84:58:b4:3b:24:66:bc:2c:ad:58: + 85:0c:62:41:fc:96:a1:ee:c9:dc:1d:51:13:ac:6e: + a1:a4:da:be:d2:6d:ef:17:cc:36:df:f5:d3:67:09: + 86:67:a0:e2:4c:88:6b:e9:d3:c0:f9:14:51:89:2f: + e2:bb:37:dc:42:10:70:a9:1c:9e:66:17:2c:b0:ce: + e9:e8:47:7b:ac:12:05:7e:d1:bf:27:b7:6f:3b:45: + c2:e8:04:d6:8e:44:22:ed:e8:bd:45:4a:c4:9b:ea: + 0d:cd:4b:f0:14:34:d0:db:d5:52:e8:6b:de:87:14: + b2:f7:ac:10:aa:4a:dd:f2:48:21:ad:cf:79:67:70: + 41:b9:1d:70:51:34:45:a8:91:ae:12:2a:a1:c3:38: + ff:33:0d:96:71:b6:63:70:5a:9e:90:b4:76:85:8f: + 8a:d8:fd:79:cc:1a:27:a4:26:02:06:72:bc:de:07: + c8:c0:83:01:82:f6:ed:b7:62:8e:7f:21:7e:4e:86: + 04:88:6e:a8:f4:9a:d6:15:47:06:70:25:bb:33:5d: + 54:27:ec:13:5b:43:33:45:36:09:e9:24:fa:49:c4: + 0e:31:0d:1a:f8:d7:92:2c:01:01:e6:ee:e8:2d:dc: + 6d:1a:5f:63:db:6d:76:31:3a:91:7f:51:df:b7:25: + 0b:b9:bd:81:9b:70:c5:31:a8:e3:da:1e:2d:c2:f1: + 05:13:ad:cb:c8:44:19:b8:5b:62:f6:7d:dd:f1:00: + da:42:e9:16:9f:45:55:e3:7d:e1:85:7e:4d:29:c8: + 0a:e5:bc:8c:98:08:6b:f1:01:e5 Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + 27:04:5B:37:51:0D:92:FD:2C:60:61:FE:4F:9C:66:20:41:2C:08:45 + X509v3 Authority Key Identifier: + C0:0A:2B:28:43:DE:5F:C9:7D:47:E5:47:9B:36:F2:65:8C:67:3B:E2 Signature Algorithm: sha256WithRSAEncryption Signature Value: - 7e:dd:64:64:92:6c:b9:41:ce:f3:e3:f8:e6:9f:c8:5b:32:39: - 8c:03:5b:5e:7e:b3:23:ca:6c:d1:99:2f:53:af:9d:3c:84:cd: - c6:ce:0a:ee:94:de:ff:a7:06:81:7e:e2:38:a5:05:39:58:22: - dc:13:83:53:e7:f8:16:cb:93:dc:cf:4b:e6:1b:9f:9e:71:ef: - ee:ba:ea:b6:68:5c:32:22:7e:54:4f:46:a6:0b:11:8f:ef:05: - 6e:d3:0b:d0:a8:be:95:23:a2:e4:e7:a8:a2:a4:7d:98:52:86: - a4:15:fb:74:7a:9a:89:23:43:20:26:3a:56:9e:a3:6e:54:02: - 76:4e:25:9c:a1:8c:03:99:e5:eb:a6:61:b4:9c:2a:b1:ed:eb: - 94:f9:14:aa:a4:c3:f0:f7:7a:03:a3:b1:f8:c0:83:79:ab:8a: - 93:7f:0a:95:08:50:ff:55:19:ac:28:a2:c8:9f:a6:77:72:a3: - da:37:a9:ff:f3:57:70:c8:65:d9:55:14:84:b4:b3:78:86:82: - da:84:2c:48:19:51:ec:9d:20:b1:4d:18:fb:82:9f:7b:a7:80: - 22:69:25:83:4d:bf:ac:31:64:f5:39:11:f1:ed:53:fb:67:ab: - 91:86:c5:4d:87:e8:6b:fe:9a:84:fe:6a:92:6b:62:c1:ae:d2: - f0:cb:06:6e:f3:50:f4:8d:6d:fa:7d:6a:1c:64:c3:98:91:da: - c9:8c:a9:79:e5:48:4c:a2:de:42:28:e8:0e:9f:52:6a:a4:e0: - c7:ac:11:9c:ba:5d:d6:84:93:56:28:f1:6d:83:aa:62:b2:b7: - 56:c6:64:d9:96:4e:97:ab:4e:8f:ba:f6:ab:b9:17:52:98:32: - 7f:b5:12:fa:39:d7:34:2a:f3:ed:40:90:6f:66:7b:b6:c1:9d: - b9:53:d0:e3:e9:69:8c:cf:7a:fd:08:0a:62:47:d4:ce:72:f7: - 6f:80:b4:1d:18:7a:ba:a2:a9:45:49:ef:9c:0b:99:89:03:ab: - 5f:7a:9d:c5:77:b7 + a0:b7:48:56:c6:b3:92:26:3f:d8:92:3c:ed:72:b5:89:ea:fd: + c9:66:da:ba:6a:8e:0d:04:ea:b2:fd:1f:e4:29:da:1e:c7:8f: + 5a:f0:88:74:dd:b3:f0:5e:a7:c4:77:13:cf:a8:19:fb:f2:2d: + ee:47:b4:0c:7c:5b:d3:dc:2f:2a:5c:bf:43:22:1c:91:d8:03: + 7d:44:90:c0:2d:fe:9e:7c:8b:ef:39:4e:b3:87:99:c8:eb:c2: + b7:cf:86:65:05:52:8c:15:b9:6a:8d:cd:e3:2a:29:d1:f5:87: + 42:11:c3:2e:42:ec:ed:26:55:8d:f3:ad:66:f4:79:72:f7:9e: + ed:bc:0c:5a:a7:74:ab:dc:57:e8:5c:99:b6:32:8f:7e:58:6e: + 70:48:ea:5d:a7:fa:b1:fc:c0:e9:50:3a:a4:53:21:e7:8b:77: + 54:2d:bb:64:1e:fc:88:86:90:c1:03:90:17:bb:3c:cf:ce:e8: + 48:32:c2:07:e5:8e:4f:93:a9:1a:e2:f5:93:a8:01:9f:30:26: + 1e:ed:b5:62:e9:25:c5:b0:32:e1:fc:bd:d6:48:b4:70:a9:e2: + cd:f6:a9:42:cb:bb:24:39:b9:34:fc:b9:cb:09:01:f0:5e:7e: + ef:b5:59:d6:88:31:a9:4c:be:7d:5b:de:4d:ec:84:1b:a1:1b: + d8:7d:83:cb:f1:04:c9:f1:f3:a4:08:05:3c:b5:96:13:1d:37: + 8a:23:83:22:86:72:17:13:5e:e8:89:06:58:cd:89:42:71:12: + e5:47:fc:f7:6e:96:28:8f:19:b9:d7:86:5b:c5:62:14:e1:5b: + 06:e7:e0:66:7e:fc:b7:9e:a9:99:14:e5:0a:d6:df:8f:b5:a2: + 1a:74:54:30:f6:f4:bf:1b:43:1d:be:4f:38:92:55:10:7b:d8: + 4f:e0:33:0f:40:2e:58:ec:9c:78:1b:43:17:b3:cb:0b:f5:34: + e2:7e:11:a1:90:b6:3c:79:6a:0b:91:ce:0b:8d:d5:60:e4:6d: + c8:2a:3d:40:6d:17 -----BEGIN CERTIFICATE----- -MIIEJDCCAowCCQDLLYCZWmlSYTANBgkqhkiG9w0BAQsFADBNMQswCQYDVQQGEwJY -WTEmMCQGA1UECgwdUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24gQ0ExFjAUBgNV -BAMMDW91ci1jYS1zZXJ2ZXIwHhcNMTgwODI5MTQyMzE2WhcNMzcxMDI4MTQyMzE2 -WjBbMQswCQYDVQQGEwJYWTEXMBUGA1UEBwwOQ2FzdGxlIEFudGhyYXgxIzAhBgNV -BAoMGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMQ4wDAYDVQQDDAVub3NhbjCC -AaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAL33ERg9PAU3+OKLSWbXVyQQ -aFzFyJKwiBOCKivngIuBDLsleWVNiYBYVNODbM0G8Mx7EYjSH7G1nirVF6eCGC3f -c3djoJPe/8vsmQTVptBsW94pb/J2XEfT6MuWwQ1JaPJLALzT5/0vdtdNwPxPgW71 -S3LZGDVr8TJ6SeDlGZ+SYs2DF+dflHmdaGHQCJruHcDHbMIAt/GpkiZ/HBq8SAex -b/JmL5P56gnB7SLtrotpSguqw9X1mFqDOI0Npqmk8cfjY1bf1aVGHHVSO2JGbYmK -GfPFx+z73TMtdKnmnGj9wT0hrh3yJ6rdHF39eCOUXoVdcUmPB8vfoPB6HLybc8oo -C475/IyA888xBvuhpHL8wEbQ1rJGt5PDIogtLbNp8hb+IqmAgJvGtYmv8fVvdLo8 -EphQTa5CzT9k+dGOOHRV42e57slxGg1t7bsjZ3nopQGxUQ92iSNTn281iLpJ75WJ -le1RszW78jSS8MymsCKhvC0kViBwTzo3WUUmMdLAMwIDAQABMA0GCSqGSIb3DQEB -CwUAA4IBgQB+3WRkkmy5Qc7z4/jmn8hbMjmMA1tefrMjymzRmS9Tr508hM3Gzgru -lN7/pwaBfuI4pQU5WCLcE4NT5/gWy5Pcz0vmG5+ece/uuuq2aFwyIn5UT0amCxGP -7wVu0wvQqL6VI6Lk56iipH2YUoakFft0epqJI0MgJjpWnqNuVAJ2TiWcoYwDmeXr -pmG0nCqx7euU+RSqpMPw93oDo7H4wIN5q4qTfwqVCFD/VRmsKKLIn6Z3cqPaN6n/ -81dwyGXZVRSEtLN4hoLahCxIGVHsnSCxTRj7gp97p4AiaSWDTb+sMWT1ORHx7VP7 -Z6uRhsVNh+hr/pqE/mqSa2LBrtLwywZu81D0jW36fWocZMOYkdrJjKl55UhMot5C -KOgOn1JqpODHrBGcul3WhJNWKPFtg6pisrdWxmTZlk6Xq06PuvaruRdSmDJ/tRL6 -Odc0KvPtQJBvZnu2wZ25U9Dj6WmMz3r9CApiR9TOcvdvgLQdGHq6oqlFSe+cC5mJ -A6tfep3Fd7c= +MIIEbzCCAtegAwIBAgIJAMstgJlaaVJhMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAgFw0xODA4MjkxNDIzMTZaGA8yNTI1MTAy +ODE0MjMxNlowWzELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4 +MSMwIQYDVQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEOMAwGA1UEAwwF +bm9zYW4wggGiMA0GCSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQDD5QDHBBTqG5g/ +OW6kVb6oTHLM4q6wX42YofPGBjWFlIeMX78A1PzRvgKLo9RXbBOb/Q89rbtsL9dd ++FcUaNP7GNLnX+MpeOnMOL6M7LzihFi0OyRmvCytWIUMYkH8lqHuydwdUROsbqGk +2r7Sbe8XzDbf9dNnCYZnoOJMiGvp08D5FFGJL+K7N9xCEHCpHJ5mFyywzunoR3us +EgV+0b8nt287RcLoBNaORCLt6L1FSsSb6g3NS/AUNNDb1VLoa96HFLL3rBCqSt3y +SCGtz3lncEG5HXBRNEWoka4SKqHDOP8zDZZxtmNwWp6QtHaFj4rY/XnMGiekJgIG +crzeB8jAgwGC9u23Yo5/IX5OhgSIbqj0mtYVRwZwJbszXVQn7BNbQzNFNgnpJPpJ +xA4xDRr415IsAQHm7ugt3G0aX2PbbXYxOpF/Ud+3JQu5vYGbcMUxqOPaHi3C8QUT +rcvIRBm4W2L2fd3xANpC6RafRVXjfeGFfk0pyArlvIyYCGvxAeUCAwEAAaNCMEAw +HQYDVR0OBBYEFCcEWzdRDZL9LGBh/k+cZiBBLAhFMB8GA1UdIwQYMBaAFMAKKyhD +3l/JfUflR5s28mWMZzviMA0GCSqGSIb3DQEBCwUAA4IBgQCgt0hWxrOSJj/Ykjzt +crWJ6v3JZtq6ao4NBOqy/R/kKdoex49a8Ih03bPwXqfEdxPPqBn78i3uR7QMfFvT +3C8qXL9DIhyR2AN9RJDALf6efIvvOU6zh5nI68K3z4ZlBVKMFblqjc3jKinR9YdC +EcMuQuztJlWN861m9Hly957tvAxap3Sr3FfoXJm2Mo9+WG5wSOpdp/qx/MDpUDqk +UyHni3dULbtkHvyIhpDBA5AXuzzPzuhIMsIH5Y5Pk6ka4vWTqAGfMCYe7bVi6SXF +sDLh/L3WSLRwqeLN9qlCy7skObk0/LnLCQHwXn7vtVnWiDGpTL59W95N7IQboRvY +fYPL8QTJ8fOkCAU8tZYTHTeKI4MihnIXE17oiQZYzYlCcRLlR/z3bpYojxm514Zb +xWIU4VsG5+Bmfvy3nqmZFOUK1t+PtaIadFQw9vS/G0Mdvk84klUQe9hP4DMPQC5Y +7Jx4G0MXs8sL9TTifhGhkLY8eWoLkc4LjdVg5G3IKj1AbRc= -----END CERTIFICATE----- diff --git a/Lib/test/certdata/pycacert.pem b/Lib/test/certdata/pycacert.pem index 0a48bf7d235..c2c8b1ecdcf 100644 --- a/Lib/test/certdata/pycacert.pem +++ b/Lib/test/certdata/pycacert.pem @@ -7,96 +7,96 @@ Certificate: Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server Validity Not Before: Aug 29 14:23:16 2018 GMT - Not After : Oct 28 14:23:16 2037 GMT + Not After : Oct 28 14:23:16 2525 GMT Subject: C=XY, O=Python Software Foundation CA, CN=our-ca-server Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (3072 bit) Modulus: - 00:d0:a0:9b:b1:b9:3b:79:ee:31:2f:b8:51:1c:01: - 75:ed:09:59:4c:ac:c8:bf:6a:0a:f8:7a:08:f0:25: - e3:25:7b:6a:f8:c4:d8:aa:a0:60:07:25:b0:cf:73: - 71:05:52:68:bf:06:93:ae:10:61:96:bc:b3:69:81: - 1a:7d:19:fc:20:86:8f:9a:68:1b:ed:cd:e2:6c:61: - 67:c7:4e:55:ea:43:06:21:1b:e9:b9:be:84:5f:c2: - da:eb:89:88:e0:42:a6:45:49:2a:d0:b9:6b:8c:93: - c9:44:3b:ca:fc:3c:94:7f:dd:70:c5:ad:d8:4b:3b: - dc:1e:f8:55:4b:b5:27:86:f8:df:b0:83:cf:f7:16: - 37:bf:13:17:34:d4:c9:55:ed:6b:16:c9:7f:ad:20: - 4e:f0:1e:d9:1b:bf:8d:ba:cd:ca:96:ef:1e:69:cb: - 51:66:61:48:0d:e8:79:a3:59:61:d4:10:8d:e0:0d: - 3b:0b:85:b6:d5:85:12:dd:a5:07:c5:4b:fa:23:fa: - 54:39:f7:97:0b:c9:44:47:1e:56:77:3c:3c:13:01: - 0b:6d:77:d6:14:ba:44:f4:53:35:56:d9:3d:b9:3e: - 35:5f:db:33:9a:4f:5a:b9:38:44:c9:32:49:fe:66: - f6:1f:1a:97:aa:ca:4c:4a:f6:b8:5e:40:7f:fb:0b: - 1d:f8:5b:a1:dc:f8:c0:2e:d0:6d:49:f5:d2:46:d4: - 90:57:fe:92:81:34:ae:2d:38:bb:a8:17:0c:e1:e5: - 3f:e2:f7:26:05:54:50:f5:64:b3:1c:6e:44:ff:6f: - a9:b4:03:96:e9:0e:c2:88:d8:72:52:90:99:c6:41: - 0f:46:90:59:b8:3e:6f:d2:e2:9e:1d:36:82:95:d3: - 58:8a:12:f3:e2:d8:0d:20:51:23:f0:90:2d:9a:3e: - 7d:26:86:b2:a7:d7:f9:28:60:03:e3:77:c7:88:04: - c9:fe:89:77:70:10:4f:c3:a0:8a:3b:f4:ab:42:7b: - e3:14:92:d8:ae:16:d6:a1:de:7d + 00:e6:ba:e2:e4:c1:c2:0c:1c:3e:62:d8:b9:5c:57: + 2e:52:b8:83:c5:88:3a:e6:9a:7a:f5:64:16:33:eb: + 37:6e:2f:7b:f3:68:03:45:65:47:5d:71:10:59:ca: + 2b:1b:00:6c:81:14:61:f4:86:59:3f:ea:fd:78:37: + 16:9d:43:f1:c4:f6:69:8c:c5:29:06:88:9e:26:22: + 04:ac:04:d8:87:34:48:39:eb:6b:f2:0b:92:aa:c3: + 6e:63:51:51:6b:c2:ad:ff:5c:c8:2f:b2:1b:9c:20: + 8a:40:3e:a2:2f:6a:ea:c8:d9:37:43:5c:dc:ed:92: + e2:d9:40:d2:61:9f:71:8a:f5:ed:39:ba:a8:5e:3e: + b5:21:63:10:d8:6f:b4:e2:11:01:0b:10:e8:bb:fb: + 62:ef:48:55:bc:f5:d2:9c:ab:68:ae:95:25:19:f2: + 97:7d:1a:dc:66:ea:88:5e:86:e4:cb:cb:69:4d:5e: + b0:a3:fb:6c:31:e4:28:60:5e:90:f1:d4:2e:10:50: + e1:85:f0:0d:5c:bd:dd:45:24:08:19:3e:1c:93:66: + 8f:2b:da:53:7d:04:1c:0e:42:c4:68:5e:a6:cd:a9: + 18:ed:a7:cd:6a:d0:d1:86:ba:90:ff:b7:4c:de:c7: + 43:24:6d:c7:1c:6b:9c:81:e7:e1:1b:57:25:90:a9: + 0e:c9:56:f3:f6:6b:5e:2d:b4:2e:40:50:9b:42:63: + d2:d6:99:1c:38:dc:cf:2b:2c:a7:72:f1:c7:5e:63: + 34:76:48:f4:3e:88:13:9e:86:16:53:2f:74:fb:87: + 01:8d:22:a4:68:33:ee:13:6c:7a:06:14:54:56:17: + 57:57:98:34:d0:0b:66:09:e3:88:09:f8:a5:15:1c: + 10:73:d0:88:50:99:5e:18:65:3b:ff:31:27:1b:5e: + c6:aa:41:fd:2d:2f:18:a7:c0:f2:ab:c7:22:b5:0b: + 69:d2:73:d1:bb:d0:1c:3d:fa:a4:35:62:cd:33:86: + c7:a0:23:0f:b9:6a:d5:d2:6d:8d Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: - F3:EC:94:8E:F2:8E:30:C4:8E:68:C2:BF:8E:6A:19:C0:C1:9F:76:65 + C0:0A:2B:28:43:DE:5F:C9:7D:47:E5:47:9B:36:F2:65:8C:67:3B:E2 X509v3 Authority Key Identifier: - F3:EC:94:8E:F2:8E:30:C4:8E:68:C2:BF:8E:6A:19:C0:C1:9F:76:65 + C0:0A:2B:28:43:DE:5F:C9:7D:47:E5:47:9B:36:F2:65:8C:67:3B:E2 X509v3 Basic Constraints: critical CA:TRUE X509v3 Key Usage: critical Digital Signature, Certificate Sign, CRL Sign Signature Algorithm: sha256WithRSAEncryption Signature Value: - 8b:00:54:72:b3:8d:eb:f3:af:34:9f:d6:60:ea:de:84:3f:8c: - 04:8f:19:a6:be:02:67:c4:63:c5:74:e3:47:37:59:83:94:06: - f1:45:19:e8:07:2f:d6:4e:4b:4f:a8:3d:c7:07:07:27:92:f4: - 7e:73:4f:8b:32:19:94:46:7a:25:c4:d9:c4:27:b0:11:63:3a: - 60:8b:85:e1:73:4f:34:3b:6b:a4:34:8c:49:8e:cd:cf:4f:b2: - 65:27:41:19:b0:fc:80:31:78:f2:73:6a:9b:7d:71:34:50:fc: - 78:a8:da:05:b4:9c:5b:3a:99:7a:6b:5d:ef:3b:d3:e9:3b:33: - 01:12:65:cf:5e:07:d8:19:af:d5:53:ea:f0:10:ac:c4:b6:26: - 3c:34:2e:74:ee:64:dd:1d:36:75:89:44:00:b0:0d:fd:2f:b3: - 01:cc:1a:8b:02:cd:6c:e8:80:82:ca:bf:82:d7:00:9d:d8:36: - 15:d2:07:37:fc:6c:73:1d:da:a8:1c:e8:20:8e:32:7a:fe:6d: - 27:16:e4:58:6c:eb:3e:f0:fe:24:52:29:71:b8:96:7b:53:4b: - 45:20:55:40:5e:86:1b:ec:c9:46:91:92:ee:ac:93:65:91:2e: - 94:b6:b6:ac:e8:a3:34:89:a4:1a:12:0d:4d:44:a5:52:ed:8b: - 91:ee:2f:a6:af:a4:95:25:f9:ce:c7:5b:a7:00:d3:93:ca:b4: - 3c:5d:4d:f7:b1:3c:cc:6d:b4:45:be:82:ed:18:90:c8:86:d1: - 75:51:50:04:4c:e8:4f:d2:d6:50:aa:75:e7:5e:ff:a1:7b:27: - 19:1c:6b:49:2c:6c:4d:6b:63:cc:3b:73:00:f3:99:26:d0:82: - 87:d3:a9:36:9c:b4:3d:b9:48:68:a8:92:f0:27:8e:0c:cd:15: - 76:42:84:80:c9:3f:a3:7e:b4:dd:d7:f8:ac:76:ba:60:97:3c: - 5a:1f:4e:de:71:ee:09:39:10:dd:31:d5:68:57:5d:1a:e5:42: - ba:0e:68:e6:45:d3 + bc:e3:56:22:03:e4:5c:b9:67:ff:94:bc:75:9c:00:85:b0:d5: + 9c:c4:c3:29:66:5e:8b:b2:a9:a6:30:86:71:1a:6b:f2:00:c5: + 82:ab:5f:50:04:2a:fb:ed:b8:4c:b9:00:1b:49:57:92:11:cd: + a2:bc:cb:0f:a8:b4:61:f8:14:ca:a0:ec:40:17:ba:55:a1:c4: + bc:a6:b2:5a:ef:f4:20:10:77:47:d0:a0:c5:58:b9:6c:b5:10: + 7b:85:4a:43:a3:fb:2c:01:b9:77:17:b0:be:a0:ee:ae:ae:4d: + 67:86:48:89:57:86:78:ea:3c:ed:f0:41:35:8d:71:68:55:f9: + f2:e9:ac:32:d4:c6:a2:ef:ec:54:e6:c4:8e:2c:fd:bd:aa:60: + 56:65:33:95:ea:10:c6:74:04:eb:2a:6e:9b:11:f6:61:00:aa: + fd:ec:f2:0b:b1:4b:11:cd:93:eb:df:98:ae:4c:b4:07:04:4a: + e5:ef:ff:52:58:75:f5:3e:a4:71:e1:4a:72:5c:a9:8f:d4:aa: + 88:f0:6a:71:b4:c3:00:5f:99:6e:d7:91:af:6c:98:0d:64:c2: + 24:c7:9e:05:11:68:5e:24:62:e3:2e:45:ec:a3:34:f2:a3:9d: + 4d:e5:32:18:2f:74:fc:11:f1:36:50:4f:a0:40:29:68:5c:43: + 4c:23:6c:5d:72:c4:ec:52:76:eb:dc:b2:bc:1f:a6:c4:06:66: + 9b:5c:c7:cc:ca:f2:d1:25:4f:de:a5:1f:8d:e4:0c:49:b6:cf: + 85:40:a1:b9:1f:c6:c7:19:15:07:63:34:93:d0:57:a0:5a:70: + ec:af:4a:1c:72:17:1d:74:a3:6c:31:45:0b:33:7a:a1:b8:46: + db:c7:0e:64:4c:6f:b7:99:04:82:43:1f:e0:59:d6:99:21:27: + 28:09:40:ae:fc:c4:23:aa:a0:0c:08:05:2a:92:1c:db:23:9e: + d1:d5:63:ae:39:13:a3:12:88:5a:43:3c:4a:6e:32:f0:84:9f: + f9:09:0c:91:e7:b8 -----BEGIN CERTIFICATE----- -MIIEgDCCAuigAwIBAgIJAMstgJlaaVJbMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +MIIEgjCCAuqgAwIBAgIJAMstgJlaaVJbMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW -MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODA4MjkxNDIzMTZaFw0zNzEwMjgx -NDIzMTZaME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUg -Rm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcjCCAaIwDQYJKoZI -hvcNAQEBBQADggGPADCCAYoCggGBANCgm7G5O3nuMS+4URwBde0JWUysyL9qCvh6 -CPAl4yV7avjE2KqgYAclsM9zcQVSaL8Gk64QYZa8s2mBGn0Z/CCGj5poG+3N4mxh -Z8dOVepDBiEb6bm+hF/C2uuJiOBCpkVJKtC5a4yTyUQ7yvw8lH/dcMWt2Es73B74 -VUu1J4b437CDz/cWN78TFzTUyVXtaxbJf60gTvAe2Ru/jbrNypbvHmnLUWZhSA3o -eaNZYdQQjeANOwuFttWFEt2lB8VL+iP6VDn3lwvJREceVnc8PBMBC2131hS6RPRT -NVbZPbk+NV/bM5pPWrk4RMkySf5m9h8al6rKTEr2uF5Af/sLHfhbodz4wC7QbUn1 -0kbUkFf+koE0ri04u6gXDOHlP+L3JgVUUPVksxxuRP9vqbQDlukOwojYclKQmcZB -D0aQWbg+b9Linh02gpXTWIoS8+LYDSBRI/CQLZo+fSaGsqfX+ShgA+N3x4gEyf6J -d3AQT8Ogijv0q0J74xSS2K4W1qHefQIDAQABo2MwYTAdBgNVHQ4EFgQU8+yUjvKO -MMSOaMK/jmoZwMGfdmUwHwYDVR0jBBgwFoAU8+yUjvKOMMSOaMK/jmoZwMGfdmUw -DwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAYYwDQYJKoZIhvcNAQELBQAD -ggGBAIsAVHKzjevzrzSf1mDq3oQ/jASPGaa+AmfEY8V040c3WYOUBvFFGegHL9ZO -S0+oPccHByeS9H5zT4syGZRGeiXE2cQnsBFjOmCLheFzTzQ7a6Q0jEmOzc9PsmUn -QRmw/IAxePJzapt9cTRQ/Hio2gW0nFs6mXprXe870+k7MwESZc9eB9gZr9VT6vAQ -rMS2Jjw0LnTuZN0dNnWJRACwDf0vswHMGosCzWzogILKv4LXAJ3YNhXSBzf8bHMd -2qgc6CCOMnr+bScW5Fhs6z7w/iRSKXG4lntTS0UgVUBehhvsyUaRku6sk2WRLpS2 -tqzoozSJpBoSDU1EpVLti5HuL6avpJUl+c7HW6cA05PKtDxdTfexPMxttEW+gu0Y -kMiG0XVRUARM6E/S1lCqdede/6F7Jxkca0ksbE1rY8w7cwDzmSbQgofTqTactD25 -SGiokvAnjgzNFXZChIDJP6N+tN3X+Kx2umCXPFofTt5x7gk5EN0x1WhXXRrlQroO -aOZF0w== +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAgFw0xODA4MjkxNDIzMTZaGA8yNTI1MTAy +ODE0MjMxNlowTTELMAkGA1UEBhMCWFkxJjAkBgNVBAoMHVB5dGhvbiBTb2Z0d2Fy +ZSBGb3VuZGF0aW9uIENBMRYwFAYDVQQDDA1vdXItY2Etc2VydmVyMIIBojANBgkq +hkiG9w0BAQEFAAOCAY8AMIIBigKCAYEA5rri5MHCDBw+Yti5XFcuUriDxYg65pp6 +9WQWM+s3bi9782gDRWVHXXEQWcorGwBsgRRh9IZZP+r9eDcWnUPxxPZpjMUpBoie +JiIErATYhzRIOetr8guSqsNuY1FRa8Kt/1zIL7IbnCCKQD6iL2rqyNk3Q1zc7ZLi +2UDSYZ9xivXtObqoXj61IWMQ2G+04hEBCxDou/ti70hVvPXSnKtorpUlGfKXfRrc +ZuqIXobky8tpTV6wo/tsMeQoYF6Q8dQuEFDhhfANXL3dRSQIGT4ck2aPK9pTfQQc +DkLEaF6mzakY7afNatDRhrqQ/7dM3sdDJG3HHGucgefhG1clkKkOyVbz9mteLbQu +QFCbQmPS1pkcONzPKyyncvHHXmM0dkj0PogTnoYWUy90+4cBjSKkaDPuE2x6BhRU +VhdXV5g00AtmCeOICfilFRwQc9CIUJleGGU7/zEnG17GqkH9LS8Yp8Dyq8citQtp +0nPRu9AcPfqkNWLNM4bHoCMPuWrV0m2NAgMBAAGjYzBhMB0GA1UdDgQWBBTACiso +Q95fyX1H5UebNvJljGc74jAfBgNVHSMEGDAWgBTACisoQ95fyX1H5UebNvJljGc7 +4jAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG9w0BAQsF +AAOCAYEAvONWIgPkXLln/5S8dZwAhbDVnMTDKWZei7KppjCGcRpr8gDFgqtfUAQq +++24TLkAG0lXkhHNorzLD6i0YfgUyqDsQBe6VaHEvKayWu/0IBB3R9CgxVi5bLUQ +e4VKQ6P7LAG5dxewvqDurq5NZ4ZIiVeGeOo87fBBNY1xaFX58umsMtTGou/sVObE +jiz9vapgVmUzleoQxnQE6ypumxH2YQCq/ezyC7FLEc2T69+Yrky0BwRK5e//Ulh1 +9T6kceFKclypj9SqiPBqcbTDAF+ZbteRr2yYDWTCJMeeBRFoXiRi4y5F7KM08qOd +TeUyGC90/BHxNlBPoEApaFxDTCNsXXLE7FJ269yyvB+mxAZmm1zHzMry0SVP3qUf +jeQMSbbPhUChuR/GxxkVB2M0k9BXoFpw7K9KHHIXHXSjbDFFCzN6obhG28cOZExv +t5kEgkMf4FnWmSEnKAlArvzEI6qgDAgFKpIc2yOe0dVjrjkToxKIWkM8Sm4y8ISf ++QkMkee4 -----END CERTIFICATE----- diff --git a/Lib/test/certdata/pycakey.pem b/Lib/test/certdata/pycakey.pem index a6bf7356f4f..0248f985545 100644 --- a/Lib/test/certdata/pycakey.pem +++ b/Lib/test/certdata/pycakey.pem @@ -1,40 +1,40 @@ -----BEGIN PRIVATE KEY----- -MIIG/gIBADANBgkqhkiG9w0BAQEFAASCBugwggbkAgEAAoIBgQDQoJuxuTt57jEv -uFEcAXXtCVlMrMi/agr4egjwJeMle2r4xNiqoGAHJbDPc3EFUmi/BpOuEGGWvLNp -gRp9Gfwgho+aaBvtzeJsYWfHTlXqQwYhG+m5voRfwtrriYjgQqZFSSrQuWuMk8lE -O8r8PJR/3XDFrdhLO9we+FVLtSeG+N+wg8/3Fje/Exc01MlV7WsWyX+tIE7wHtkb -v426zcqW7x5py1FmYUgN6HmjWWHUEI3gDTsLhbbVhRLdpQfFS/oj+lQ595cLyURH -HlZ3PDwTAQttd9YUukT0UzVW2T25PjVf2zOaT1q5OETJMkn+ZvYfGpeqykxK9rhe -QH/7Cx34W6Hc+MAu0G1J9dJG1JBX/pKBNK4tOLuoFwzh5T/i9yYFVFD1ZLMcbkT/ -b6m0A5bpDsKI2HJSkJnGQQ9GkFm4Pm/S4p4dNoKV01iKEvPi2A0gUSPwkC2aPn0m -hrKn1/koYAPjd8eIBMn+iXdwEE/DoIo79KtCe+MUktiuFtah3n0CAwEAAQKCAYAD -iUK0/k2ZRqXJHXKBKy8rWjYMHCj3lvMM/M3g+tYWS7i88w00cIJ1geM006FDSf8i -LxjatvFd2OCg9ay+w8LSbvrJJGGbeXAQjo1v7ePRPttAPWphQ8RCS+8NAKhJcNJu -UzapZ13WJKfL2HLw1+VbziORXjMlLKRnAVDkzHMZO70C5MEQ0EIX+C6zrmBOl2HH -du6LPy8crSaDQg8YxFCI7WWnvRKp+Gp8aIfYnR+7ifT1qr5o9sEUw8GAReyooJ3a -yJ9uBUbcelO8fNjEABf9xjx+jOmOVsQfig2KuBEi0qXlQSpilZfUdYJhtNke9ADu -Hui6MBn04D4RIzeKXV+OLjiLwqkJyNlPuxJ2EGpIHNMcx3gpjXIApAwc47BQwLKJ -VhMWMXS0EWhCLtEzf5UrbMNX+Io3J7noEUu6jxmJV1BKhrnlYeoo4JryN0DUpkSb -rOAOJLOkpfj7+gvqmWI4MT6SQXSr6BK+3m4J5bVSq4pej9uG5NR3Utghi5hF7DEC -gcEA3cYNPYPFSTj9YAR3GUZvwUPVL3ZEFcwjrIeg87JhuZOH/hSQ33SgeEoAtaqL -cLbimj7YzUYx3FOUCp/7yK/bAF1dhAbFab1yZE46Qv2Vi4e+/KEBBftqxyJl5KyV -vc/HE1dXZGZIO1X5Z5MX8nO3rz/YayiozYVmMibrbHxgTEDC4BrbWtPJQNkflWEb -FXNjkm0s2+J3kFANpL94NUKMGtArxQV3hWydGN8wS3Fn7LDnHDoM5mOt/naeKRES -fwwpAoHBAPDTKsKs2LEe4YFzO1EClycDelppjxh5pHSnzTWSq40aKx533SG4aLyI -DmghzoA3OmY0xpAy1WpT9FeiDNbYpiFCH3qBkArQR2QCu+WGUQ9tDoeN0C2Dje4e -Yix49BjcGSWzSNvh+tU9PzRc/9eVBMAQuaCm3yNEL+Z7hFTzkrCWK23+jP/OzIIC -XhnKdOveIYVAjlVgv8CoWIy3xhwXyqPAcstcPmlv9sDAYn37Ot7rGIS7e0WyQxvg -gxnOxFzKNQKBwQDOPOn/NNV5HKh0bHKdbKVs4zoT4zW515etUIvbVR4QSCSFonZ/ -d6PreVZjmvAFp+3fZ2aSrx6bOJZJszGhFfjhw/G9X9aiWO1SXnVL6yrxERIJOWkM -ORy5h0GegOjYFauaTvUUhxHRLEi9i0sPy5EcRpFqReuFBPNe3Fa/EoMzJl6TriYj -tyRHTCNU9XMMZbxJZYH8EgUCjY/Cj9SoIvTL0p+Bn23hBHqrsJLm9dWhhXnHBC0O -68/Y/lJi+l9rCtECgcEAt6PfTJovl0j8HxF23vyBtK9TQtSR2NERlh9LPZn9lViq -Hs66YndT7sg1bDSzWlRDBSMjc1xAH5erkJOzBLYqYNwiUvGvnH9coSfwjkMRVxkL -ZlS+taZGuZiTtmP5h2d3CaegXIQDGU5d/xkXwxYQjEF0u8vkBel+OVxg+cLPTjcF -IRhl/r98dXtGtJYM+LvnhcxHfVWMg2YcOBn/SPbfgGVFZEuQECjf2fYaZQUJzGkr -xjOM+gXIZN6cOjbQyA0tAoHADgR5/bMbcf6Jk0w56c/khFZz/pusne5cjXw5a6qq -fClAqnqjGBpkRxs7HoCR3aje0Pd0pCS93a6Wiqneo4x4HDrpo+pWR2KGAAF4MeO3 -3K94hncmiLAiZo8iqULLKCqJW2EGB2b7QzGpY7jCPiI1g80KuYPesf4ZohSfrr1w -DoqGoNrcIVdVmUgX47lLqIiWarbbDRY0Am9j58dovmNINYr5wCYGbeh2RuUmHr4u -E2bb0CdekSHf05HPiF9QpK1z +MIIG/AIBADANBgkqhkiG9w0BAQEFAASCBuYwggbiAgEAAoIBgQDmuuLkwcIMHD5i +2LlcVy5SuIPFiDrmmnr1ZBYz6zduL3vzaANFZUddcRBZyisbAGyBFGH0hlk/6v14 +NxadQ/HE9mmMxSkGiJ4mIgSsBNiHNEg562vyC5Kqw25jUVFrwq3/XMgvshucIIpA +PqIvaurI2TdDXNztkuLZQNJhn3GK9e05uqhePrUhYxDYb7TiEQELEOi7+2LvSFW8 +9dKcq2iulSUZ8pd9Gtxm6ohehuTLy2lNXrCj+2wx5ChgXpDx1C4QUOGF8A1cvd1F +JAgZPhyTZo8r2lN9BBwOQsRoXqbNqRjtp81q0NGGupD/t0zex0Mkbccca5yB5+Eb +VyWQqQ7JVvP2a14ttC5AUJtCY9LWmRw43M8rLKdy8cdeYzR2SPQ+iBOehhZTL3T7 +hwGNIqRoM+4TbHoGFFRWF1dXmDTQC2YJ44gJ+KUVHBBz0IhQmV4YZTv/MScbXsaq +Qf0tLxinwPKrxyK1C2nSc9G70Bw9+qQ1Ys0zhsegIw+5atXSbY0CAwEAAQKCAYBf +4VWcPjBHHA2Iwgr1Jn1nfqmzklL3tUZXZwoa9SoJrc3Sbmy9j8LCP9PNnEehZuGw +GipClPnNp/dA15OcMrnrYYKnLt9Hico+inBqk3Dvbnh9KSmoYcrHD4N13jr5juMD +dSjzOQ5kKNmKrPx0u/dpE2r1oUdlql5+bYN/ceSbHGtCTCDfWSun/iTn7DO8pdhL +IvGz/Fk2mlaWuYiV9lz//5Z1W+w73sesNNYKgf/d+F9/+VNqMXbanLdypJmTBNp9 +eoS3eLk3ycoiGHCnVNm28IjeElFkUOJKVXY39BoMbS/x1MUHjuZDxdOcEaCV9iOr +adg7d91srwJlHGGX3IkC+08J5OJZtfTEMcB3i9g2omOxLAFfapeRxZdD6zMdxWUG +PUNIiy7UKaHe7JJ+fftnk0QBu5DzyrdP0fryFNz9nySot+gzVlU9idTMnIyE73UJ +foPowKpmxDOYNw0ZGXBIP/lMRNXkQM765P1NJpCN0Ud+kBl7GdFMkI++GFjiH3kC +gcEA+VIsOliXn5xfU2zXZb2q/bPPlCBDw/EzIelRTFVqUC2kL3oh4NlDNWAFgcfX +rQUCCoY9rEKYdTQI0w47K4AkJYajnDG2zHj6fkaPcLweyEDa9Hw2ezDDiwz2uwH2 +4HW+JBZ0I7rRuSRXTSJrzFYYVU6tsr54Kl85l2bmL8V2XVYj4ko9uddsyH0F1Ejt +BYPBghr01uEHsO3nIcNMl10fS9eXdhMTr/haXdDxpFXjR5AO/ouQEyGrXk6LPJpn +rUE7AoHBAOzpN8lQDQlf0JcYFBvFd7JdrhpbaIdATmT0NM/Mh+4B8qy5qnIJBdeQ +9T46sHmHzvhtGHmnP+dckKJi1Y/8Lt6noWNYDRnmrDol046ce2Nj8gxDGBT1kG4n +GBoKkA1Gj5N5yWMB8cGTVzQ+tViRMkn9+jmxRaNsW1Y0j7lMtVz0hJOUscbUALV1 +fYGH7zPiqfHoKoz7UuB2OlCKOH0+V96lAO4EgKMQNcEKMs9+Tg13fHTAyodxy/CY +tjUcKpef1wKBwBE24EDjDw0BMf/Dmxe2QdEkkieLFsK3q60iu+9GUoHYtOZmS2KH +/cD4sUilsLmMh/iMDkQPkRE+l4FjESjOvzAsHK3TLOjvTXRckNja1FFFURjiXqyg +0E+QhJSi7RXQa2F4f2pcItDitnhn8QN5ylJRjWKzDf729jYC78/KlYKaSP393Ecx +nZw2LanboynnT/wYumD/xpUrx/Kn1mj5EAkfiKCpbomO30Zs/9I17+xoAPEIV9lK +UNfBGpIDozbuMwKBwFlZmAWf4FrRvSzPEv5qWjt2I2yjXufrs+VVSPm6LOXx7CGC +oKsDhiWH8UZ4AgjD1KZTFvECyBItEgt8dQkp1k95L1/1XHORURFZJNHbaJnSnv5K +67Ez8DXrHqbrpuqq2wmG3BIwMIqOVExK/kAZ+rp3REEv/5CkFEqN5kq/iIM3YSz7 +3pSbbm0Bk8UfjHKoIOowYqPrQZWQYWvwxV9O/PrmhlQ+dHmLaoqUmxcwjqV7k//A +mmG85GqoXcfoCJRI3wKBwBdnxBzg17TMFufuvX2Bc/M9MqL3+vlwH6SDdr+2yYKA +hiD8Ur2OwtDGHnV4m3NeA/Guyz32H4CzvFAnpzlvMow/dvfp9JUcpdeidhIBZy8V +D7VODIiCyyTAb3g1LK0+HTEHAVRFihbNXhub/P6NckFXw0MJJQOvNNsYQnJTUngY +oxqdt1HeAujEwBrRSfrOGE2K8FVJ/MYf4PmxTIocICBk1/BmNHsUeJ5yFUDBweh5 +UJN6yp5PiGwvW8WFl4waXw== -----END PRIVATE KEY----- diff --git a/Lib/test/certdata/revocation.crl b/Lib/test/certdata/revocation.crl index 431a831c4f1..a10586d1f79 100644 --- a/Lib/test/certdata/revocation.crl +++ b/Lib/test/certdata/revocation.crl @@ -1,14 +1,14 @@ -----BEGIN X509 CRL----- -MIICJjCBjwIBATANBgkqhkiG9w0BAQsFADBNMQswCQYDVQQGEwJYWTEmMCQGA1UE +MIICKDCBkQIBATANBgkqhkiG9w0BAQsFADBNMQswCQYDVQQGEwJYWTEmMCQGA1UE CgwdUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24gQ0ExFjAUBgNVBAMMDW91ci1j -YS1zZXJ2ZXIXDTIzMTEyNTA0MjEzNloXDTQzMDEyNDA0MjEzNlqgDjAMMAoGA1Ud -FAQDAgEAMA0GCSqGSIb3DQEBCwUAA4IBgQDMZ4XLQlzUrqBbszEq9I/nXK3jN8/p -VZ2aScU2le0ySJqIthe0yXEYuoFu+I4ZULyNkCA79baStIl8/Lt48DOHfBVv8SVx -ZqF7/fdUZBCLJV1kuhuSSknbtNmja5NI4/lcRRXrodRWDMcOmqlKbAC6RMQz/gMG -vpewGPX1oj5AQnqqd9spKtHbeqeDiyyWYr9ZZFO/433lP7GdsoriTPggYJJMWJvs -819buE0iGwWf+rTLB51VyGluhcz2pqimej6Ra2cdnYh5IztZlDFR99HywzWhVz/A -2fwUA91GR7zATerweXVKNd59mcgF4PZWiXmQMwcE0qQOMqMmAqYPLim1mretZsAs -t1X+nDM0Ak3sKumIjteQF7I6VpSsG4NCtq23G8KpNHnBZVOt0U065lQEvx0ZmB94 -1z7SzjfSZMVXYxBjSXljwuoc1keGpNT5xCmHyrOIxaHsmizzwNESW4dGVLu7/JfK -w40uGbwH09w4Cfbwuo7w6sRWDWPnlW2mkoc= +YS1zZXJ2ZXIXDTI0MTAwODExNTExMloYDzI0MDgwMTI5MTE1MTEyWqAOMAwwCgYD +VR0UBAMCAQAwDQYJKoZIhvcNAQELBQADggGBAKN4S2g4wCeU1fO5TSckAwxgdzdh +pY28f4musnQt7l37MzB2gmJVDSCZfQyrUnfSEST15WEY7CZVyTlbsu6gYKK53Yej +j3ORBfGUgzaz62Hs8di7SrHDzWUlNCFa47YFWDmtj96KTX1AItnpkCCE58Wfpivp +Hu+YINFpi/2vI2nvP/xcfvgT3dXek9kyz+2jHmadxcn2VerSBZZ9fiZk/k4NzgoI +JdiSswtN1c5GelHQfftwRXbWqsp6TvgHC5MagDuHh5Bj7/DftI7nCy0IT5GnP8lS +ZqmXUMpa8zbtSNSTIk0XepmypNW8HHMQbfJp0y7yOQ4pPyXICrjYTg7wKpODRcm3 +BRN89vvNfCszMU41glVfQG+2Po5uAMTl1hX8WYSj0+Xxrdg+wgJead4S5Sq3CgMT +bKsH2Dqh43L8BTxuxzLQyduK0gKSl8vlN7a9Bzm3IXYlyk+kKSyo4jP8XK79pj1k +1JglMFM9jpoMF2VmNjiROtVEl2tbDGwlvpjWYQ== -----END X509 CRL----- diff --git a/Lib/test/certdata/ssl_cert.pem b/Lib/test/certdata/ssl_cert.pem index 427948252b7..6db52404942 100644 --- a/Lib/test/certdata/ssl_cert.pem +++ b/Lib/test/certdata/ssl_cert.pem @@ -1,27 +1,27 @@ -----BEGIN CERTIFICATE----- -MIIEgzCCAuugAwIBAgIUU+FIM/dUbCklbdDwNPd2xemDAEwwDQYJKoZIhvcNAQEL +MIIEhTCCAu2gAwIBAgIUTxhwRX6zBQc3+l3imDklEbqcDpIwDQYJKoZIhvcNAQEL BQAwXzELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYD VQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjESMBAGA1UEAwwJbG9jYWxo -b3N0MB4XDTIzMTEyNTA0MjEzNloXDTQzMDEyNDA0MjEzNlowXzELMAkGA1UEBhMC -WFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRob24gU29m -dHdhcmUgRm91bmRhdGlvbjESMBAGA1UEAwwJbG9jYWxob3N0MIIBojANBgkqhkiG -9w0BAQEFAAOCAY8AMIIBigKCAYEAzXTIl1su11AGu6sDPsoxqcRGyAX0yjxIcswF -vj+eW/fBs2GcBby95VEOKpJPKRYYB7fAEAjAKK59zFdsDX/ynxPZLqyLQocBkFVq -tclhCRZu//KZND+uQuHSx3PjGkSvK/nrGjg5T0bkM4SFeb0YdLb+0aDTKGozUC82 -oBAilNcrFz1VXpEF0qUe9QeKQhyd0MaW5T1oSn+U3RAj2MXm3TGExyZeaicpIM5O -HFlnwUxsYSDZo0jUj342MbPOZh8szZDWi042jdtSA3i8uMSplEf4O8ZPmX0JCtrz -fVjRVdaKXIjrhMNWB8K44q6AeyhqJcVHtOmPYoHDm0qIjcrurt0LZaGhmCuKimNd -njcPxW0VQmDIS/mO5+s24SK+Mpznm5q/clXEwyD8FbrtrzV5cHCE8eNkxjuQjkmi -wW9uadK1s54tDwRWMl6DRWRyxoF0an885UQWmbsgEB5aRmEx2L0JeD0/q6Iw1Nta -As8DG4AaWuYMrgZXz7XvyiMq3IxVAgMBAAGjNzA1MBQGA1UdEQQNMAuCCWxvY2Fs -aG9zdDAdBgNVHQ4EFgQUl2wd7iWE1JTZUVq2yFBKGm9N36owDQYJKoZIhvcNAQEL -BQADggGBAF0f5x6QXFbgdyLOyeAPD/1DDxNjM68fJSmNM/6vxHJeDFzK0Pja+iJo -xv54YiS9F2tiKPpejk4ujvLQgvrYrTQvliIE+7fUT0dV74wZKPdLphftT9uEo1dH -TeIld+549fqcfZCJfVPE2Ka4vfyMGij9hVfY5FoZL1Xpnq/ZGYyWZNAPbkG292p8 -KrfLZm/0fFYAhq8tG/6DX7+2btxeX4MP/49tzskcYWgOjlkknyhJ76aMG9BJ1D7F -/TIEh5ihNwRTmyt023RBz/xWiN4xBLyIlpQ6d5ECKmFNFr0qnEui6UovfCHUF6lZ -qcAQ5VFQQ2CayNlVmQ+UGmWIqANlacYWBt7Q6VqpGg24zTMec1/Pqd6X07ScSfrm -MAtywrWrU7p1aEkN5lBa4n/XKZHGYMjor/YcMdF5yjdSrZr274YYO1pafmTFwRwH -5o16c8WPc0aPvTFbkGIFT5ddxYstw+QwsBtLKE2lJ4Qfmxt0Ew/0L7xkbK1BaCOo -EGD2IF7VDQ== +b3N0MCAXDTI0MTAwODExNTExMloYDzI0MDgwMTI5MTE1MTEyWjBfMQswCQYDVQQG +EwJYWTEXMBUGA1UEBwwOQ2FzdGxlIEFudGhyYXgxIzAhBgNVBAoMGlB5dGhvbiBT +b2Z0d2FyZSBGb3VuZGF0aW9uMRIwEAYDVQQDDAlsb2NhbGhvc3QwggGiMA0GCSqG +SIb3DQEBAQUAA4IBjwAwggGKAoIBgQDwcIAYm12nmQTGB3caFn7alDe3LSliEfNC +2ZTR+5sh1eucQPbzgFM5SR4soKGElwI68Eg7g1kwqu3zmrI/FAiQI/RrUHyBZiEt +nFBPM44vY02XjUlJ9nBtgP7QjpRz6ZP0z1DrrojpYLVwg9cR0khTqD5cg2jvTB05 +yL9lQNk295/rMuueC9FaAQ+Y5la0ub7Lbe8gkYPotYliGx5qqCQmsXeTsxCpvQD7 +u0vBF9/nxlwywwzKGXabcN9YQhSECCC4c+eYjqoRgvAaF48xEnokxenzBIqZ82BR +kFo+zfNR+VhJRctNiZ6Ppa1Ise1H3LjZMDfY1S89QOLYsFXUp8L7ZMVE27Bej+Sq +4wEJ3soK/k1kr0YauqJ0lCEKkUPD9OeNHmczeakjj11tgtctsYbwgDSUYGA3w+DC +KD1aSfeuR3Hj89cSsVRrRrPFFih46Tr2PpTNoK6MtPH3RPJKV+Db8E1V2mXmNE0M +Lg6ramSHsD9iZXTLhG2JO+/876k3N3kCAwEAAaM3MDUwFAYDVR0RBA0wC4IJbG9j +YWxob3N0MB0GA1UdDgQWBBR459BlAel5MqCtG0DrvVtMZlQfuTANBgkqhkiG9w0B +AQsFAAOCAYEAaTFWjK/LFzOo+0TWqTTj4WC4N3I8JFrHnlqFJlpchYTW2z92SU1G +iEzFjWzuDNjp5KM9BqlmGtzXZvy6MItGkYsjPRdPVU0rbCmyTho6y77kTyiEG12V +UAJ1in3FOQfDwLPcp7wQRgCQq3iZlv7pwXp2Lm5fzu8kZPnxmTVdiKQun9Ps7uKq +BoM0fM2K14MxVO4Wc0SERnaPszE7xAhkIcs+NRT/gHYdTBlPhao83S3LOOdtCqCP +pNUOEaShlwI5bVsDPUXNX/eS0MYFNlsYTb5rCxK8jf3W3KNjKTOzN94pHiQOhpkg +xMPPi3m03/9oKTVXBtHI2A+u3ukheKE6sBXCLdv/GEs9zYI49zmpQxNWz5EOumWL +k+W/vPv7cD6LeHxxp+nCbEJi1gZtYb3gMY1sLkMNxcOu0QHTqHfme+k7VKWm8anO +3ogGdGtPuPAD/qjMwg3ChSDLl5Ur/E9UPlD4yM/7KtUD7mLv+jbddA62EiA9MxVB +t+yt7pOwOA66 -----END CERTIFICATE----- diff --git a/Lib/test/certdata/ssl_key.passwd.pem b/Lib/test/certdata/ssl_key.passwd.pem index 6ab7d57d003..e60a66c2798 100644 --- a/Lib/test/certdata/ssl_key.passwd.pem +++ b/Lib/test/certdata/ssl_key.passwd.pem @@ -1,42 +1,42 @@ -----BEGIN ENCRYPTED PRIVATE KEY----- -MIIHbTBXBgkqhkiG9w0BBQ0wSjApBgkqhkiG9w0BBQwwHAQIsc9l0YPybNICAggA -MAwGCCqGSIb3DQIJBQAwHQYJYIZIAWUDBAEqBBDxb9ekR9MERvIff73hFLc6BIIH -ENhkFePApZj7ZqpjBltINRnaZhu8sEfG1/y3ejDBOa5Sq3C/UPykPfJh0IXsraAB -STZO22UQEDpJzDnf1aLCo2cJpdz4Mr+Uj8OUdPiX83OlhC36gMrkgSYUdhSFQEas -MLiBnXU6Z5Mv1Lxe7TJrnMyA4A8JYXXu5XVTErJrC0YT6iCPQh7eAoEtml9a/tJM -OPg6kn58zmzVDp8LAau4Th1yhdD/cUQM09wg2i5JHLeC9akD+CkNlujVoAirLMTh -xoMXTy2dkv/lIwI9QVx6WE/VKIngBAPIi3Q+YCIm0PaTgWj5U10C8j4t7kW2AEZK -z82+vDOpLRGLo/ItNCO9F/a9e4PK4xxwFCOfR80tQNhs5gjKnbDz5IQv2p+pUfUX -u+AIO0rBb3M9Yya1MC2pc5VLAeQ3UF6YPrNyNjoDsQOytY3YtRVyxiKW72QzeUcX -Vpc3U6u8ZyHhkxK6bMv3dkPHGW1MOBd9/U5z+9lhHOfCGFStIQ9M8N48ZCWEGyty -oZT3UApxgqiBAi1h14ZyagA2mjsMNtTmmkSa3v26WUfrwnjm7LD1/0Vm+ptBOFH2 -CkP/aAvr8Ie+ehWobXGpqwB6rlOAwdpPrePtEZiZtdt58anmCquRgE5GIYtVz30f -flRABM8waJ196RDGkNAmDA3p/sqHy4vbsIOMl8faZ3QxvGVZlPbUEwPhiTIetA5Q -95fT/uIcuBLfpbaN23j/Av3LiJAeABSmGZ+dA+NXC5UMvuX8COyBU0YF2V6ofpIu -gP3UC7Tn4yV3Pbes81LEDCskaN6qVRil47l0G+dNcEHVkrGKcSaRCN+joBSCbuin -Rol34ir9azh8DqHRKdVlLlzTmDQcOwmi0Vx0ASgBXx4UI3IfK45gLJVoz6dkUz+3 -GIPrnh5cw2DvIgIApwmuCQUXPbWZwUW0zuyzhtny9W6S72GUE/P5oUCV+kGYBsup -FNiAyR9+n/xUuzB5HqIosj4rX+M4il4Ovt+KaCO6/COi+YjAO/9EnSttu8OTxsXl -wvgblsT7Y1d+iUfmIVNGtbc5NX46ktrbGiqgPX7oR7YDy5/+FQlnPS1YL0ThUiAC -2RbItu6b0uUyfu2jfWaGqy+SiRZ81rLwKPU3vJSEPfooVcJTG49EE006ZC4TvRzu -fNkId+P+BxvhEpUM4+VKzfzViEPuzR1u/DuwLAavS7nr5qb+zaUq+Fte5vDQmjjC -fflT8hS0BGpYEGndeZT4k+mZunHgs3NVUQ4/HW0nflf1j6qAn4+yIB79dH9d/ubt -RyBG29K+rN0TI/kH9BQZfsAcbnmhpT/ud0mJfeHZ0Lknn6mdJ/k4LXN0T1IlLKz3 -cSleOWY3zjKaOsbuju1o5IiVIr+AF/w+M4nzzDX6DDVpBPAt9iUnDGqjh6mJ3QWQ -CyCJDLNP0X8rZ8va2KOPorIBhmfDwJKEtIoXkb2hqWURTE0chC444QqiMsMXsX6+ -mOmiWGkdBFnEpGITISFTGERCjEfqOgTMweCANpquiLymJXgDURL603N2WexSgwnu -Gy1Ws1cA+1cT65ZLqjSqayZ6WdQvsKBBAnGW5LbwBhoCkX0vahs5nZiw0KnskP60 -wNMnyxaS1SuDJ65n+vuLUl7WeysRyz10RWliYZFiUE7jIXfWeYGonAo4eyCEeV/f -HInxxpswsg/na8BGBPMsx2SfBIiIvSIT4VNxHrL3sIfDrnb2HH/ut/oSLBgSKzY5 -DdkPz309kMM5dqnHANAgRrtVhqzLQE3kNGZ9mO/X1FAyXx8eB7NSeB6ysD8CAHvm -lkyfsGTzVsnuWWpeHqplds0wx5+XouVtFRI5J3RGa39mbpM1hMyIbS0O24CBKW6K -7n2UunbABwepL1hSa4e01OPdz4Zx/oayOevTtlfVqh68cEEc6ePdzf7z69pjot7B -eqlNaqa1POOmkuygL+fiP1BAR3rGEoQKXqb+6JjzLM9CnhCQHHPR2UdqukkEYwsa -bh9CU8AlfAJ19KFDria4JZXtl8LLMLLqWIO8fmQx7VqkEkEkl8jecO8YMaZTzFEb -bW7QtIZ1qHWH0UIHH3Qlav72NJTKvGIbtp1JNrLdsHcYNcojLZkEeA83UPaiTB2R -udltVUd016cktRVzLOKrust8kzPq3iSjpoIXFyFqIYHvWxGHgc7qD5gVBlazqSsV -qudDv+0PCBjLWLjS6HkFI8BfyXd3ME2wvSmTzSSgSh4nVJNNrZ/RVTtQ5MLVcdh0 -sJ3qsq2Pokf61XXjsTiQorX+cgI9zF6zETXHvnLf9FL+G/VSlcLUsQ0wC584qwQt -OSASYTbM79xgmjRmolZOptcYXGktfi2C4iq6V6zpFJuNMVgzZ+SbaQw9bvzUo2jG -VMwrTuQQ+fsAyn66WZvtkSGAdp58+3PNq31ZjafJXBzN +MIIHdTBfBgkqhkiG9w0BBQ0wUjAxBgkqhkiG9w0BBQwwJAQQMMUN+qWFiwTbIjjb +cLEtYQICCAAwDAYIKoZIhvcNAgkFADAdBglghkgBZQMEASoEEGGzvsc6e5Zn2M9S +sdoMbaIEggcQ0mQYuMcujHtFfRKLmfclrf/dHOYgnVYc2c9MSR3jfoOwnTdBR1Op +MUuO4ukqrxGkrJBy+MLb1oPEGrnvn39pdBMZx4CBmdwX0jDq36UL9EU5/m071mT+ +Xoe/z/RHtrtmbpA9XKu7n/5gLi9jg57qorss8r5MaPHghm2ZOqaQZIOOMCF+4gNU +Cmp/NHL9TUM/sfQpaw9YM4VuTRyHNO3OiDKaCCz2romFrgM1TYz/I5/n5MMR543J +dsyIFzef3jHYfaXTl97v9ibGVpIUCKfHgYlQf41H+Me890AP4HaS0e2y+73/UwjB +4YSzPVm+lCiBWhFDbMuRmPY3YDQOA+TV06oCO8/aMamPRsryl8g7iOQAIz4hQdoP +ZTXtSZ+F68YmfPjKAj3NNQosvLzfuSfxZ9HqyvZ/0w1eead7pdSs7uvCEvN2Zcvk +a9EWy1bM895zD3DwrxqGnSYbitXouIOf9zHsl2lRUK44XmvgTHoNaJFIVyzVNgQr +e1YJM7LbEvErspa6cZTz7DCR44g9cYBi393XEt5yl4NCtq2PFq762pNqpsQqt5/Y +pLX5w12npf+OZ1WqJla89FabqLaJ1blmDEgfko0XriofW5gWLByvhYINtt+u5x/5 +QdTya2gWBxI2K+sAeGIt913RBAus8hLEDnVGDX/FEthz1OOKLXp0G6HQmqUom5PU +/O36GszhWIz1Q/SpMfZkdHfEfbJFCx0mpaJZTlC01hg+OpnwWxUvO2dxrK46xAzd +/obsVQRLOuyTxjIrQd+vVZRIcOroTHVUbh7qV2NV1sdQbsKnc4mYK8jjHsb/v/O3 +lcOFJX2V55z7FdEOi7qKKn3l5j6SNmdMGlGde16692VCdYo4Wjs2ufkYryJoJQ7L +O3AeoXVrpc2JwKj6GFKN3Bw6A3XEXTShWdHS8DAuHdbLBhHESeb/qA76qNInw/od +PXtsRNBWaS87rh7GGWrO2ULHBk0k5hLa9puXgAZVh5NcrVuujWM2spGPHGJWhvDt +ffOJAe0Kra8b1CHOC9aRUuHnapuu/mAD+XZTAt/UUNq0hdTDO1VTxL4lHpZjmWJ3 +OvmTp1HpA8FpSmzSBw6H0beVT9LXDbkSax86E2YQJPcC9IjQARoaL+7ba1XdXeSQ +k796zrq4YD1SB09qRVVar8nBSHUYAzG2gbAWgAQgOjwtC5nT1Que3kI7weG2nFZ4 +B1MfBmpEqPwMs8O5hurdYmfpBRoudivKrK5si+LB7og1IwVD2IS8mm55/Kmnugf4 +YCEiGu5s9F+njqEZBV7UPAabaFph6Iw3Jfmcg0YUZV60hY6O3EieonfwVBBKWBQp +K0SUs24Ld80B3oYWuqYI+MQlSprskRKFBfx7hp2PsQU9jAVPc8NohmfM5IJAFBwo +rCqAOsRsrRZnVoaRP89X50VuBg6ROiu/cmI5sqou3u30Pndzmlbcg8ekWj+1qrur +prva6uPc0oiCgFTMksFnhKvlHErIm+ceGvNSib+rWNomaL5cjpgnqAG6lZMP7N+p +QvH+ABchP+fJyyl/XuaOLmcXDSkIcadfjJSHACwmGRas3unfOYEyBg29oPu6PNlW +jXFJb7OzWaQfFNcCnHzvYGTvmrbg7VqBVqmVRYX3r6WNDJeaSwDJpxYNNQlTugbm +FOlD9JTeHyZ17rrqiCitbQwBVBpWOhBEIkFD+3JL1ZdGjyc3rcvbERLr/UN649vS +FcMOLiEvFjFdirq6Pe9fx/VD1GrIzXaEhvyCePSeoV6eILm7SSFTVCm+JMtiBlBi +ZjDxlUhQGuMg3IwZRqjfUR90wo3QWe5XOgM9mJa5qY/Yaa9YpFEJcd4mYnXcPLIz +eY8lqGAhROwJhGRhQoTQXucsSEhAkUBpRiE7UCN8OjezvAeFn0+6XyHeST+QZenX +ixAKJ27lFn/njvL4sDocd7ZvXpb3P5ZxCRakMnjunQQyjtUlJmPrNjs+ZlPenLuh +UA3Oj5d96dDqzgZNLxbDHKr6B+CMApBrwUDcum09PYgJ5xZ/Hrct4iSa57Gi528L +l0dcVgPHd80oIn/vyhjVYkkXNMrRhTJIgLuh0KsddZ7/8Xvxma9W8hNQ1sYwE0Yp +RqLgMRdFpTSN8hKUTfRvjzbrW4nQ2XVqAdyM6PEkdrPCBZczdGisp9oMF6woI/pA +ZKNxdUr0DG3tVBJ7z6qfdp2j+yHvhqQt+ohmk3YldPKpXfYz20ZJ52URE8vbCj2K +NZ/MVEACbg6FDgNQ1bIKD61pKL72+SjyKW1wQfIkeqMfOBI8BGclp7BL6dGRrVHN +PLg2j9gsgZ3XsiJOFtJ6Q3UABkeUrbRvAFQXPM8+keWU4VP/99dCJTvnEQTM6O8U +gz2NA4j1ZEOmB5L+hmf2gW+xgW6eeMjylj5JdeAliNaUqIhOju30vo4= -----END ENCRYPTED PRIVATE KEY----- diff --git a/Lib/test/certdata/ssl_key.pem b/Lib/test/certdata/ssl_key.pem index ee927210511..1ad9e5e2b83 100644 --- a/Lib/test/certdata/ssl_key.pem +++ b/Lib/test/certdata/ssl_key.pem @@ -1,40 +1,40 @@ -----BEGIN PRIVATE KEY----- -MIIG/QIBADANBgkqhkiG9w0BAQEFAASCBucwggbjAgEAAoIBgQDNdMiXWy7XUAa7 -qwM+yjGpxEbIBfTKPEhyzAW+P55b98GzYZwFvL3lUQ4qkk8pFhgHt8AQCMAorn3M -V2wNf/KfE9kurItChwGQVWq1yWEJFm7/8pk0P65C4dLHc+MaRK8r+esaODlPRuQz -hIV5vRh0tv7RoNMoajNQLzagECKU1ysXPVVekQXSpR71B4pCHJ3QxpblPWhKf5Td -ECPYxebdMYTHJl5qJykgzk4cWWfBTGxhINmjSNSPfjYxs85mHyzNkNaLTjaN21ID -eLy4xKmUR/g7xk+ZfQkK2vN9WNFV1opciOuEw1YHwrjiroB7KGolxUe06Y9igcOb -SoiNyu6u3QtloaGYK4qKY12eNw/FbRVCYMhL+Y7n6zbhIr4ynOebmr9yVcTDIPwV -uu2vNXlwcITx42TGO5COSaLBb25p0rWzni0PBFYyXoNFZHLGgXRqfzzlRBaZuyAQ -HlpGYTHYvQl4PT+rojDU21oCzwMbgBpa5gyuBlfPte/KIyrcjFUCAwEAAQKCAYAO -M1r0+TCy4Z1hhceu5JdLql0RELZTbxi71IW2GVwW87gv75hy3hGLAs/1mdC+YIBP -MkBka1JqzWq0/7rgcP5CSAMsInFqqv2s7fZ286ERGXuZFbnInnkrNsQUlJo3E9W+ -tqKtGIM/i0EVHX0DRdJlqMtSjmjh43tB+M1wAUV+n6OjEtJue5wZK+AIpBmGicdP -qZY+6IBnm8tcfzPXFRCoq7ZHdIu0jxnc4l2MQJK3DdL04KoiStOkSl8xDsI+lTtq -D3qa41LE0TY8X2jJ/w6KK3cUeK7F4DQYs+kfCKWMVPpn0/5u6TbC1F7gLvkrseph -7cIgrruNNs9iKacnR1w3U72R+hNxHsNfo4RGHFa192p/Mfc+kiBd5RNR/M9oHdeq -U6T/+KM+QyF5dDOyonY0QjwfAcEx+ZsV72nj8AerjM907I6dgHo/9YZ2S1Dt/xuG -ntD+76GDzmrOvXmmpF0DsTn+Wql7AC4uzaOjv6PVziqz03pR61RpjPDemyJEWMkC -gcEA7BkGGX3enBENs3X6BYFoeXfGO/hV7/aNpA6ykLzw657dqwy2b6bWLiIaqZdZ -u0oiY6+SpOtavkZBFTq4bTVD58FHL0n73Yvvaft507kijpYBrxyDOfTJOETv+dVG -XiY8AUSAE6GjPi0ebuYIVUxoDnMeWDuRJNvTck4byn1hJ1aVlEhwXNxt/nAjq48s -5QDuR6Z9F8lqEACRYCHSMQYFm35c7c1pPsHJnElX8a7eZ9lT7HGPXHaf/ypMkOzo -dvJNAoHBAN7GhDomff/kSgQLyzmqKqQowTZlyihnReapygwr8YpNcqKDqq6VlnfH -Jl1+qtSMSVI0csmccwJWkz1WtSjDsvY+oMdv4gUK3028vQAMQZo+Sh7OElFPFET3 -UmL+Nh73ACPgpiommsdLZQPcIqpWNT5NzO+Jm5xa+U9ToVZgQ7xjrqee5NUiMutr -r7UWAz7vDWu3x7bzYRRdUJxU18NogGbFGWJ1KM0c67GUXu2E7wBQdjVdS78UWs+4 -XBxKQkG2KQKBwQCtO+M82x122BB8iGkulvhogBjlMd8klnzxTpN5HhmMWWH+uvI1 -1G29Jer4WwRNJyU6jb4E4mgPyw7AG/jssLOlniy0Jw32TlIaKpoGXwZbJvgPW9Vx -tgnbDsIiR3o9ZMKMj42GWgike4ikCIc+xzRmvdMbHIHwUJfCfEtp9TtPGPnh9pDz -og3XLsMNg52GXnt3+VI6HOCE41XH+qj2rZt5r2tSVXEOyjQ7R5mOzSeFfXJVwDFX -v/a/zHKnuB0OAdUCgcBLrxPTEaqy2eMPdtZHM/mipbnmejRw/4zu7XYYJoG7483z -SlodT/K7pKvzDYqKBVMPm4P33K/x9mm1aBTJ0ZqmL+a9etRFtEjjByEKuB89gLX7 -uzTb7MrNF10lBopqgK3KgpLRNSZWWNXrtskMJ5eVICdkpdJ5Dyst+RKR3siEYzU9 -+yxxAFpeQsqB8gWORva/RsOR8yNjIMS3J9fZqlIdGA8ktPr0nEOyo96QQR5VdACE -5rpKI2cqtM6OSegynOkCgcAnr2Xzjef6tdcrxrQrq0DjEFTMoCAxQRa6tuF/NYHV -AK70Y4hBNX84Bvym4hmfbMUEuOCJU+QHQf/iDQrHXPhtX3X2/t8M+AlIzmwLKf2o -VwCYnZ8SqiwSaWVg+GANWLh0JuKn/ZYyR8urR79dAXFfp0UK+N39vIxNoBisBf+F -G8mca7zx3UtK2eOW8WgGHz+Y20VZy0m/nkNekd1ZTXoSGhL+iN4XsTRn1YQIn69R -kNdcwhtZZ3dpChUdf+w/LIc= +MIIG/gIBADANBgkqhkiG9w0BAQEFAASCBugwggbkAgEAAoIBgQDwcIAYm12nmQTG +B3caFn7alDe3LSliEfNC2ZTR+5sh1eucQPbzgFM5SR4soKGElwI68Eg7g1kwqu3z +mrI/FAiQI/RrUHyBZiEtnFBPM44vY02XjUlJ9nBtgP7QjpRz6ZP0z1DrrojpYLVw +g9cR0khTqD5cg2jvTB05yL9lQNk295/rMuueC9FaAQ+Y5la0ub7Lbe8gkYPotYli +Gx5qqCQmsXeTsxCpvQD7u0vBF9/nxlwywwzKGXabcN9YQhSECCC4c+eYjqoRgvAa +F48xEnokxenzBIqZ82BRkFo+zfNR+VhJRctNiZ6Ppa1Ise1H3LjZMDfY1S89QOLY +sFXUp8L7ZMVE27Bej+Sq4wEJ3soK/k1kr0YauqJ0lCEKkUPD9OeNHmczeakjj11t +gtctsYbwgDSUYGA3w+DCKD1aSfeuR3Hj89cSsVRrRrPFFih46Tr2PpTNoK6MtPH3 +RPJKV+Db8E1V2mXmNE0MLg6ramSHsD9iZXTLhG2JO+/876k3N3kCAwEAAQKCAYAK +Ap0KqTlCd4fv2LK4NtSMNByHt00gRKAMmfNstJ12UKoxBLFjXOXaHjWv5PYkh4bz +vjo7pBHMCWnDuR6Pqr1ahuyvpRex6XcbJ4VebsaOKYO6+gphlm2C2ZqCQ1Vh6Akd +aZ40Wb1gfgK/zvVezBLvzLLf9iahw9j5pWZ2iDci5zdUuvd9Sn+qUB3+nyRf/NW5 +MXgBsp07zIcOOxPOm/Z5V+0jDJL2hiRq1pbmUKClTShcgqtfJKU//niF+6ZQAuiJ +LBPaKIdPXyxLYnkyq2IgjPU0ZwxzdP0a2k72kvImd25Daj7elhGr3++IR+nFzt6h +vqflOfmKDF3zZPyUVI3YXjxo/FrOwGbLMHuuHBgE9txH/mOl1gByrxP+Ax18i3Bf +spSLeUvtaf/w/MopyspPoJBRbAM06PUHQI2v9xq3BZL/gHe2CdJPds2WzpaaVFG4 +oJWNrE3s6CowLqUkqzB7LqJ4ReZ6xe6SpkRotdmVknlIKgDenTFeEUEEVyBiFQEC +gcEA/F1GAaBG0e9vB+AOHZ96SLlZVzObSBYq2kVwUhhGItNnyU9c3fWPIrGREKQa +lw5dsvjl58ij5uEtJoPZf5BsJ0q6xHAs/kKxfpNfZAeoKAV96Z6MVcY+6WOyGjPF +aQo+GgSrCPIciW//WXZrWI1t0M2G0vZ5CFNohnKod+cSgV03PAActlyM2H+r7dtm +MpAD3EPWeeA75saKj/x0SOzuL/wzXKR8BZ6CINZ6r61Tcbk2mDwOHPhUrHeCwjoU +nhy5AoHBAPPnP2FSXFCPXD1Z1hFInCFgm41j7LEyBTVLlnqUrRk7i18fi/WcwwLH ++XvM5DcONY/V3sh7a3tZeHN1P70tRxLE0oO51D4tP5im/oZ6L+hszSYXX7lCbJSR +tni6nU1dssW3nmswfUn01Oh+B0rBGon3RQB6x4beTAW0piVxg9Ic2HYucS1Scrqw +afiFQ5KWklnMYJKInPFzlCwMdgBCuue1dZoJstU9nLQALNNSpGXB2X0+7j9D/qkz +Caw5MfgQwQKBwQDzdCvP78XCSuBq0XvsmefG9n+4fwGDFld6v9gualpmyFjsPJKT +UYwm5PPUAOvh46sCt9hatRVg6sO6zyFoTXP4p7/rN2hAVSiTuiog/r369elVEW3C +ZYBVeKbdXipIPehRA0XYWHCtKY1Fydae07kn4M37AGkcXhKM+VmKajFQ+RMK3/TS +/A+n3+qFiM1bY9FFkW/7nRVMeSY850dq/p59TihibA91AEf6084BYg0IvatsSys2 +SV6uDpDnPE6dhYkCgcBECtAwq1RbmRLnfqdsnPAJk7Txhd3jNQwk6RhqzA1aS7U+ +7UMTWw9AOF+OPQOxpEInBUgob931RGmI9D263eXFA6mi2/Ws/tyODpBVHcM9uRSm +OsEWosQ90kSwe4ckrS4RYH9OcfGR7z5yOa55GVP5B0V1s8r0AhH9SX9MVNWsiSWO +GriyJx0gndSCY1MNkvnzGUQbvQbjiRXeD//fZL5Vo9bSCUCdopmT0bSvo49/X8v3 +19WJSsPBmh5psG8TQEECgcEA64CqZpPux35LeLQsKe0fYeNfAncqiaIoRbAvxKCi +SQf27SD8HK+sfvhvYY7bP7TMEeM7B/O2/AqBQQP0UARIGJg2AknBQT0A7//yJu+o +v4FHy2XKh+RMAx7QrdvnQ4CfrjvjQIaAcN1HrdTKWwgQZZImRf57nUCMm82ktZ2k +vYEJTXMkT8CY0DSeGtPmX5ynk7cauHTdZrkPGhZ3Hr6GAFomOammnnytv2wc+5FA +Ap+d65UgF4KjGY4rtsS+jOHn -----END PRIVATE KEY----- diff --git a/Lib/test/cjkencodings/big5-utf8.txt b/Lib/test/cjkencodings/big5-utf8.txt new file mode 100644 index 00000000000..a0a534a964d --- /dev/null +++ b/Lib/test/cjkencodings/big5-utf8.txt @@ -0,0 +1,9 @@ +如何在 Python 中使用既有的 C library? + 在資訊科技快速發展的今天, 開發及測試軟體的速度是不容忽視的 +課題. 為加快開發及測試的速度, 我們便常希望能利用一些已開發好的 +library, 並有一個 fast prototyping 的 programming language 可 +供使用. 目前有許許多多的 library 是以 C 寫成, 而 Python 是一個 +fast prototyping 的 programming language. 故我們希望能將既有的 +C library 拿到 Python 的環境中測試及整合. 其中最主要也是我們所 +要討論的問題就是: + diff --git a/Lib/test/cjkencodings/big5.txt b/Lib/test/cjkencodings/big5.txt new file mode 100644 index 00000000000..f4424959e91 --- /dev/null +++ b/Lib/test/cjkencodings/big5.txt @@ -0,0 +1,9 @@ +pb Python ϥάJ C library? +@bTާֳtoi, }oδճn骺t׬Oe +D. [ֶ}oδժt, ڭ̫K`ƱQΤ@Ǥw}on +library, æ@ fast prototyping programming language i +Ѩϥ. ثe\\hh library OH C g, Python O@ +fast prototyping programming language. Gڭ̧ƱNJ +C library Python ҤդξX. 䤤̥Dn]Oڭ̩ +nQתDNO: + diff --git a/Lib/test/cjkencodings/big5hkscs-utf8.txt b/Lib/test/cjkencodings/big5hkscs-utf8.txt new file mode 100644 index 00000000000..f744ce9ae08 --- /dev/null +++ b/Lib/test/cjkencodings/big5hkscs-utf8.txt @@ -0,0 +1,2 @@ +𠄌Ě鵮罓洆 +ÊÊ̄ê êê̄ diff --git a/Lib/test/cjkencodings/big5hkscs.txt b/Lib/test/cjkencodings/big5hkscs.txt new file mode 100644 index 00000000000..81c42b3503d --- /dev/null +++ b/Lib/test/cjkencodings/big5hkscs.txt @@ -0,0 +1,2 @@ +E\sڍ +fb diff --git a/Lib/test/cjkencodings/cp949-utf8.txt b/Lib/test/cjkencodings/cp949-utf8.txt new file mode 100644 index 00000000000..5655e385176 --- /dev/null +++ b/Lib/test/cjkencodings/cp949-utf8.txt @@ -0,0 +1,9 @@ +똠방각하 펲시콜라 + +㉯㉯납!! 因九月패믤릔궈 ⓡⓖ훀¿¿¿ 긍뒙 ⓔ뎨 ㉯. . +亞영ⓔ능횹 . . . . 서울뤄 뎐학乙 家훀 ! ! !ㅠ.ㅠ +흐흐흐 ㄱㄱㄱ☆ㅠ_ㅠ 어릨 탸콰긐 뎌응 칑九들乙 ㉯드긐 +설릌 家훀 . . . . 굴애쉌 ⓔ궈 ⓡ릘㉱긐 因仁川女中까즼 +와쒀훀 ! ! 亞영ⓔ 家능궈 ☆上관 없능궈능 亞능뒈훀 글애듴 +ⓡ려듀九 싀풔숴훀 어릨 因仁川女中싁⑨들앜!! ㉯㉯납♡ ⌒⌒* + diff --git a/Lib/test/cjkencodings/cp949.txt b/Lib/test/cjkencodings/cp949.txt new file mode 100644 index 00000000000..16549aa5e49 --- /dev/null +++ b/Lib/test/cjkencodings/cp949.txt @@ -0,0 +1,9 @@ +c氢 ݶ + +!! Вp ިR ѵ . . +䬿Ѵ . . . . ʫR ! ! !. + ٤_  O h O +j ʫR . . . . ֚f ѱ ސtƒO  +;R ! ! 䬿 ʫɱ ߾ ɱŴ 䬴ɵR ۾֊ +޷ ǴR  Ĩ!! ҡ* + diff --git a/Lib/test/cjkencodings/euc_jisx0213-utf8.txt b/Lib/test/cjkencodings/euc_jisx0213-utf8.txt new file mode 100644 index 00000000000..9a56a2e18bd --- /dev/null +++ b/Lib/test/cjkencodings/euc_jisx0213-utf8.txt @@ -0,0 +1,8 @@ +Python の開発は、1990 年ごろから開始されています。 +開発者の Guido van Rossum は教育用のプログラミング言語「ABC」の開発に参加していましたが、ABC は実用上の目的にはあまり適していませんでした。 +このため、Guido はより実用的なプログラミング言語の開発を開始し、英国 BBS 放送のコメディ番組「モンティ パイソン」のファンである Guido はこの言語を「Python」と名づけました。 +このような背景から生まれた Python の言語設計は、「シンプル」で「習得が容易」という目標に重点が置かれています。 +多くのスクリプト系言語ではユーザの目先の利便性を優先して色々な機能を言語要素として取り入れる場合が多いのですが、Python ではそういった小細工が追加されることはあまりありません。 +言語自体の機能は最小限に押さえ、必要な機能は拡張モジュールとして追加する、というのが Python のポリシーです。 + +ノか゚ ト゚ トキ喝塀 𡚴𪎌 麀齁𩛰 diff --git a/Lib/test/cjkencodings/euc_jisx0213.txt b/Lib/test/cjkencodings/euc_jisx0213.txt new file mode 100644 index 00000000000..51e9268ca98 --- /dev/null +++ b/Lib/test/cjkencodings/euc_jisx0213.txt @@ -0,0 +1,8 @@ +Python γȯϡ1990 ǯ鳫ϤƤޤ +ȯԤ Guido van Rossum ϶ѤΥץߥ󥰸ABCפγȯ˻äƤޤABC ϼѾŪˤϤޤŬƤޤǤ +ΤᡢGuido ϤŪʥץߥ󥰸γȯ򳫻Ϥѹ BBS Υǥȡ֥ƥ ѥפΥեǤ Guido ϤθPythonפ̾Ťޤ +Τ褦طʤޤ줿 Python θ߷פϡ֥ץפǡֽưספȤɸ˽֤Ƥޤ +¿ΥץȷϸǤϥ桼ͥ褷ƿʵǽǤȤƼ礬¿ΤǤPython ǤϤäٹɲä뤳ȤϤޤꤢޤ +켫ΤεǽϺǾ¤˲ɬפʵǽϳĥ⥸塼Ȥɲä롢ȤΤ Python ΥݥꥷǤ + +Τ ȥ ԏ diff --git a/Lib/test/cjkencodings/euc_jp-utf8.txt b/Lib/test/cjkencodings/euc_jp-utf8.txt new file mode 100644 index 00000000000..7763250ebbe --- /dev/null +++ b/Lib/test/cjkencodings/euc_jp-utf8.txt @@ -0,0 +1,7 @@ +Python の開発は、1990 年ごろから開始されています。 +開発者の Guido van Rossum は教育用のプログラミング言語「ABC」の開発に参加していましたが、ABC は実用上の目的にはあまり適していませんでした。 +このため、Guido はより実用的なプログラミング言語の開発を開始し、英国 BBS 放送のコメディ番組「モンティ パイソン」のファンである Guido はこの言語を「Python」と名づけました。 +このような背景から生まれた Python の言語設計は、「シンプル」で「習得が容易」という目標に重点が置かれています。 +多くのスクリプト系言語ではユーザの目先の利便性を優先して色々な機能を言語要素として取り入れる場合が多いのですが、Python ではそういった小細工が追加されることはあまりありません。 +言語自体の機能は最小限に押さえ、必要な機能は拡張モジュールとして追加する、というのが Python のポリシーです。 + diff --git a/Lib/test/cjkencodings/euc_jp.txt b/Lib/test/cjkencodings/euc_jp.txt new file mode 100644 index 00000000000..9da6b5d83da --- /dev/null +++ b/Lib/test/cjkencodings/euc_jp.txt @@ -0,0 +1,7 @@ +Python γȯϡ1990 ǯ鳫ϤƤޤ +ȯԤ Guido van Rossum ϶ѤΥץߥ󥰸ABCפγȯ˻äƤޤABC ϼѾŪˤϤޤŬƤޤǤ +ΤᡢGuido ϤŪʥץߥ󥰸γȯ򳫻Ϥѹ BBS Υǥȡ֥ƥ ѥפΥեǤ Guido ϤθPythonפ̾Ťޤ +Τ褦طʤޤ줿 Python θ߷פϡ֥ץפǡֽưספȤɸ˽֤Ƥޤ +¿ΥץȷϸǤϥ桼ͥ褷ƿʵǽǤȤƼ礬¿ΤǤPython ǤϤäٹɲä뤳ȤϤޤꤢޤ +켫ΤεǽϺǾ¤˲ɬפʵǽϳĥ⥸塼Ȥɲä롢ȤΤ Python ΥݥꥷǤ + diff --git a/Lib/test/cjkencodings/euc_kr-utf8.txt b/Lib/test/cjkencodings/euc_kr-utf8.txt new file mode 100644 index 00000000000..16c37412b6a --- /dev/null +++ b/Lib/test/cjkencodings/euc_kr-utf8.txt @@ -0,0 +1,7 @@ +◎ 파이썬(Python)은 배우기 쉽고, 강력한 프로그래밍 언어입니다. 파이썬은 +효율적인 고수준 데이터 구조와 간단하지만 효율적인 객체지향프로그래밍을 +지원합니다. 파이썬의 우아(優雅)한 문법과 동적 타이핑, 그리고 인터프리팅 +환경은 파이썬을 스크립팅과 여러 분야에서와 대부분의 플랫폼에서의 빠른 +애플리케이션 개발을 할 수 있는 이상적인 언어로 만들어줍니다. + +☆첫가끝: 날아라 쓔쓔쓩~ 닁큼! 뜽금없이 전홥니다. 뷁. 그런거 읎다. diff --git a/Lib/test/cjkencodings/euc_kr.txt b/Lib/test/cjkencodings/euc_kr.txt new file mode 100644 index 00000000000..f68dd350289 --- /dev/null +++ b/Lib/test/cjkencodings/euc_kr.txt @@ -0,0 +1,7 @@ + ̽(Python) , α׷ Դϴ. ̽ +ȿ ȿ üα׷ +մϴ. ̽ () Ÿ, ׸ +ȯ ̽ ũð о߿ κ ÷ +ø̼ ִ ̻ ݴϴ. + +ù: ƶ ԤФԤԤФԾ~ ԤҤŭ! ԤѤݾ ԤȤϴ. ԤΤ. ׷ ԤѤ. diff --git a/Lib/test/cjkencodings/gb18030-utf8.txt b/Lib/test/cjkencodings/gb18030-utf8.txt new file mode 100644 index 00000000000..2060d2593eb --- /dev/null +++ b/Lib/test/cjkencodings/gb18030-utf8.txt @@ -0,0 +1,15 @@ +Python(派森)语言是一种功能强大而完善的通用型计算机程序设计语言, +已经具有十多年的发展历史,成熟且稳定。这种语言具有非常简捷而清晰 +的语法特点,适合完成各种高层任务,几乎可以在所有的操作系统中 +运行。这种语言简单而强大,适合各种人士学习使用。目前,基于这 +种语言的相关技术正在飞速的发展,用户数量急剧扩大,相关的资源非常多。 +如何在 Python 中使用既有的 C library? + 在資訊科技快速發展的今天, 開發及測試軟體的速度是不容忽視的 +課題. 為加快開發及測試的速度, 我們便常希望能利用一些已開發好的 +library, 並有一個 fast prototyping 的 programming language 可 +供使用. 目前有許許多多的 library 是以 C 寫成, 而 Python 是一個 +fast prototyping 的 programming language. 故我們希望能將既有的 +C library 拿到 Python 的環境中測試及整合. 其中最主要也是我們所 +要討論的問題就是: +파이썬은 강력한 기능을 지닌 범용 컴퓨터 프로그래밍 언어다. + diff --git a/Lib/test/cjkencodings/gb18030.txt b/Lib/test/cjkencodings/gb18030.txt new file mode 100644 index 00000000000..5d1f6dca232 --- /dev/null +++ b/Lib/test/cjkencodings/gb18030.txt @@ -0,0 +1,15 @@ +PythonɭһֹǿƵͨͼԣ +ѾʮķչʷȶԾзdzݶ +﷨ص㣬ʺɸָ߲񣬼еIJϵͳ +СԼ򵥶ǿʺϸʿѧϰʹáĿǰ +ԵؼڷٵķչûصԴdzࡣ + Python ʹüе C library? +YӍƼٰlչĽ, _lyԇܛwٶDzݺҕ +n}. ӿ_lyԇٶ, ҂㳣ϣһЩ_lõ +library, Kһ fast prototyping programming language +ʹ. ĿǰSS library C , Python һ +fast prototyping programming language. ҂ϣ܌е +C library õ Python ĭhМyԇ. ҪҲ҂ +ҪӑՓĆ}: +51332131 760463 858635 3195 0930 435755 5509899304 292599. + diff --git a/Lib/test/cjkencodings/gb2312-utf8.txt b/Lib/test/cjkencodings/gb2312-utf8.txt new file mode 100644 index 00000000000..efb7d8f95cd --- /dev/null +++ b/Lib/test/cjkencodings/gb2312-utf8.txt @@ -0,0 +1,6 @@ +Python(派森)语言是一种功能强大而完善的通用型计算机程序设计语言, +已经具有十多年的发展历史,成熟且稳定。这种语言具有非常简捷而清晰 +的语法特点,适合完成各种高层任务,几乎可以在所有的操作系统中 +运行。这种语言简单而强大,适合各种人士学习使用。目前,基于这 +种语言的相关技术正在飞速的发展,用户数量急剧扩大,相关的资源非常多。 + diff --git a/Lib/test/cjkencodings/gb2312.txt b/Lib/test/cjkencodings/gb2312.txt new file mode 100644 index 00000000000..1536ac10b9e --- /dev/null +++ b/Lib/test/cjkencodings/gb2312.txt @@ -0,0 +1,6 @@ +PythonɭһֹǿƵͨͼԣ +ѾʮķչʷȶԾзdzݶ +﷨ص㣬ʺɸָ߲񣬼еIJϵͳ +СԼ򵥶ǿʺϸʿѧϰʹáĿǰ +ԵؼڷٵķչûصԴdzࡣ + diff --git a/Lib/test/cjkencodings/gbk-utf8.txt b/Lib/test/cjkencodings/gbk-utf8.txt new file mode 100644 index 00000000000..75bbd31ec5a --- /dev/null +++ b/Lib/test/cjkencodings/gbk-utf8.txt @@ -0,0 +1,14 @@ +Python(派森)语言是一种功能强大而完善的通用型计算机程序设计语言, +已经具有十多年的发展历史,成熟且稳定。这种语言具有非常简捷而清晰 +的语法特点,适合完成各种高层任务,几乎可以在所有的操作系统中 +运行。这种语言简单而强大,适合各种人士学习使用。目前,基于这 +种语言的相关技术正在飞速的发展,用户数量急剧扩大,相关的资源非常多。 +如何在 Python 中使用既有的 C library? + 在資訊科技快速發展的今天, 開發及測試軟體的速度是不容忽視的 +課題. 為加快開發及測試的速度, 我們便常希望能利用一些已開發好的 +library, 並有一個 fast prototyping 的 programming language 可 +供使用. 目前有許許多多的 library 是以 C 寫成, 而 Python 是一個 +fast prototyping 的 programming language. 故我們希望能將既有的 +C library 拿到 Python 的環境中測試及整合. 其中最主要也是我們所 +要討論的問題就是: + diff --git a/Lib/test/cjkencodings/gbk.txt b/Lib/test/cjkencodings/gbk.txt new file mode 100644 index 00000000000..8788f8a2dc4 --- /dev/null +++ b/Lib/test/cjkencodings/gbk.txt @@ -0,0 +1,14 @@ +PythonɭһֹǿƵͨͼԣ +ѾʮķչʷȶԾзdzݶ +﷨ص㣬ʺɸָ߲񣬼еIJϵͳ +СԼ򵥶ǿʺϸʿѧϰʹáĿǰ +ԵؼڷٵķչûصԴdzࡣ + Python ʹüе C library? +YӍƼٰlչĽ, _lyԇܛwٶDzݺҕ +n}. ӿ_lyԇٶ, ҂㳣ϣһЩ_lõ +library, Kһ fast prototyping programming language +ʹ. ĿǰSS library C , Python һ +fast prototyping programming language. ҂ϣ܌е +C library õ Python ĭhМyԇ. ҪҲ҂ +ҪӑՓĆ}: + diff --git a/Lib/test/cjkencodings/hz-utf8.txt b/Lib/test/cjkencodings/hz-utf8.txt new file mode 100644 index 00000000000..7c11735c1f1 --- /dev/null +++ b/Lib/test/cjkencodings/hz-utf8.txt @@ -0,0 +1,2 @@ +This sentence is in ASCII. +The next sentence is in GB.己所不欲,勿施於人。Bye. diff --git a/Lib/test/cjkencodings/hz.txt b/Lib/test/cjkencodings/hz.txt new file mode 100644 index 00000000000..f882d463447 --- /dev/null +++ b/Lib/test/cjkencodings/hz.txt @@ -0,0 +1,2 @@ +This sentence is in ASCII. +The next sentence is in GB.~{<:Ky2;S{#,NpJ)l6HK!#~}Bye. diff --git a/Lib/test/cjkencodings/iso2022_jp-utf8.txt b/Lib/test/cjkencodings/iso2022_jp-utf8.txt new file mode 100644 index 00000000000..7763250ebbe --- /dev/null +++ b/Lib/test/cjkencodings/iso2022_jp-utf8.txt @@ -0,0 +1,7 @@ +Python の開発は、1990 年ごろから開始されています。 +開発者の Guido van Rossum は教育用のプログラミング言語「ABC」の開発に参加していましたが、ABC は実用上の目的にはあまり適していませんでした。 +このため、Guido はより実用的なプログラミング言語の開発を開始し、英国 BBS 放送のコメディ番組「モンティ パイソン」のファンである Guido はこの言語を「Python」と名づけました。 +このような背景から生まれた Python の言語設計は、「シンプル」で「習得が容易」という目標に重点が置かれています。 +多くのスクリプト系言語ではユーザの目先の利便性を優先して色々な機能を言語要素として取り入れる場合が多いのですが、Python ではそういった小細工が追加されることはあまりありません。 +言語自体の機能は最小限に押さえ、必要な機能は拡張モジュールとして追加する、というのが Python のポリシーです。 + diff --git a/Lib/test/cjkencodings/iso2022_jp.txt b/Lib/test/cjkencodings/iso2022_jp.txt new file mode 100644 index 00000000000..fc398d64ad2 --- /dev/null +++ b/Lib/test/cjkencodings/iso2022_jp.txt @@ -0,0 +1,7 @@ +Python $B$N3+H/$O!"(B1990 $BG/$4$m$+$i3+;O$5$l$F$$$^$9!#(B +$B3+H/e$NL\E*$K$O$"$^$jE,$7$F$$$^$;$s$G$7$?!#(B +$B$3$N$?$a!"(BGuido $B$O$h$j$E$1$^$7$?!#(B +$B$3$N$h$&$JGX7J$+$i@8$^$l$?(B Python $B$N8@8l@_7W$O!"!V%7%s%W%k!W$G!V=,F@$,MF0W!W$H$$$&L\I8$K=EE@$,CV$+$l$F$$$^$9!#(B +$BB?$/$N%9%/%j%W%H7O8@8l$G$O%f!<%6$NL\@h$NMxJX@-$rM%@h$7$F?'!9$J5!G=$r8@8lMWAG$H$7$Fl9g$,B?$$$N$G$9$,!"(BPython $B$G$O$=$&$$$C$?>.:Y9)$,DI2C$5$l$k$3$H$O$"$^$j$"$j$^$;$s!#(B +$B8@8l<+BN$N5!G=$O:G>.8B$K2!$5$(!"I,MW$J5!G=$O3HD%%b%8%e!<%k$H$7$FDI2C$9$k!"$H$$$&$N$,(B Python $B$N%]%j%7!<$G$9!#(B + diff --git a/Lib/test/cjkencodings/iso2022_kr-utf8.txt b/Lib/test/cjkencodings/iso2022_kr-utf8.txt new file mode 100644 index 00000000000..d5c9d6eeeb2 --- /dev/null +++ b/Lib/test/cjkencodings/iso2022_kr-utf8.txt @@ -0,0 +1,7 @@ +◎ 파이썬(Python)은 배우기 쉽고, 강력한 프로그래밍 언어입니다. 파이썬은 +효율적인 고수준 데이터 구조와 간단하지만 효율적인 객체지향프로그래밍을 +지원합니다. 파이썬의 우아(優雅)한 문법과 동적 타이핑, 그리고 인터프리팅 +환경은 파이썬을 스크립팅과 여러 분야에서와 대부분의 플랫폼에서의 빠른 +애플리케이션 개발을 할 수 있는 이상적인 언어로 만들어줍니다. + +☆첫가끝: 날아라 쓩~ 큼! 금없이 전니다. 그런거 다. diff --git a/Lib/test/cjkencodings/iso2022_kr.txt b/Lib/test/cjkencodings/iso2022_kr.txt new file mode 100644 index 00000000000..2cece21c5dd --- /dev/null +++ b/Lib/test/cjkencodings/iso2022_kr.txt @@ -0,0 +1,7 @@ +$)C!] FD@L=c(Python)@: 9h?l1b =10m, 0-7BGQ GA7N1W7!9V >p>n@T4O4Y. FD@L=c@: +H?@2@{@N 0mF(iPd:)GQ 9.9}0z 5?@{ E8@LGN, 1W8.0m @NEMGA8.FC +H/0f@: FD@L=c@; =:E)83FC0z ?)7/ :P>_?!<-?M 4k:N:P@G GC7'F{?!<-@G :|8% +>VGC8.DI@Lp>n7N 885i>nA]4O4Y. + +!YC90!3!: 3/>F6s >1~ E-! 1]>x@L @|4O4Y. 1W710E 4Y. diff --git a/Lib/test/cjkencodings/johab-utf8.txt b/Lib/test/cjkencodings/johab-utf8.txt new file mode 100644 index 00000000000..5655e385176 --- /dev/null +++ b/Lib/test/cjkencodings/johab-utf8.txt @@ -0,0 +1,9 @@ +똠방각하 펲시콜라 + +㉯㉯납!! 因九月패믤릔궈 ⓡⓖ훀¿¿¿ 긍뒙 ⓔ뎨 ㉯. . +亞영ⓔ능횹 . . . . 서울뤄 뎐학乙 家훀 ! ! !ㅠ.ㅠ +흐흐흐 ㄱㄱㄱ☆ㅠ_ㅠ 어릨 탸콰긐 뎌응 칑九들乙 ㉯드긐 +설릌 家훀 . . . . 굴애쉌 ⓔ궈 ⓡ릘㉱긐 因仁川女中까즼 +와쒀훀 ! ! 亞영ⓔ 家능궈 ☆上관 없능궈능 亞능뒈훀 글애듴 +ⓡ려듀九 싀풔숴훀 어릨 因仁川女中싁⑨들앜!! ㉯㉯납♡ ⌒⌒* + diff --git a/Lib/test/cjkencodings/johab.txt b/Lib/test/cjkencodings/johab.txt new file mode 100644 index 00000000000..067781b785a --- /dev/null +++ b/Lib/test/cjkencodings/johab.txt @@ -0,0 +1,9 @@ +wba \ũa + +s!! gÚ zٯٯٯ w ѕ . . + string -> td identity. s = repr(td) - self.assertTrue(s.startswith('datetime.')) + self.assertStartsWith(s, 'datetime.') s = s[9:] td2 = eval(s) self.assertEqual(td, td2) @@ -1115,6 +1132,85 @@ def test_delta_non_days_ignored(self): dt2 = dt - delta self.assertEqual(dt2, dt - days) + def test_strptime(self): + inputs = [ + # Basic valid cases + (date(1998, 2, 3), '1998-02-03', '%Y-%m-%d'), + (date(2004, 12, 2), '2004-12-02', '%Y-%m-%d'), + + # Edge cases: Leap year + (date(2020, 2, 29), '2020-02-29', '%Y-%m-%d'), # Valid leap year date + + # bpo-34482: Handle surrogate pairs + (date(2004, 12, 2), '2004-12\ud80002', '%Y-%m\ud800%d'), + (date(2004, 12, 2), '2004\ud80012-02', '%Y\ud800%m-%d'), + + # Month/day variations + (date(2004, 2, 1), '2004-02', '%Y-%m'), # No day provided + (date(2004, 2, 1), '02-2004', '%m-%Y'), # Month and year swapped + + # Different day-month-year formats + (date(2004, 12, 2), '02/12/2004', '%d/%m/%Y'), # Day/Month/Year + (date(2004, 12, 2), '12/02/2004', '%m/%d/%Y'), # Month/Day/Year + + # Different separators + (date(2023, 9, 24), '24.09.2023', '%d.%m.%Y'), # Dots as separators + (date(2023, 9, 24), '24-09-2023', '%d-%m-%Y'), # Dashes + (date(2023, 9, 24), '2023/09/24', '%Y/%m/%d'), # Slashes + + # Handling years with fewer digits + (date(127, 2, 3), '0127-02-03', '%Y-%m-%d'), + (date(99, 2, 3), '0099-02-03', '%Y-%m-%d'), + (date(5, 2, 3), '0005-02-03', '%Y-%m-%d'), + + # Variations on ISO 8601 format + (date(2023, 9, 25), '2023-W39-1', '%G-W%V-%u'), # ISO week date (Week 39, Monday) + (date(2023, 9, 25), '2023-268', '%Y-%j'), # Year and day of the year (Julian) + ] + for expected, string, format in inputs: + with self.subTest(string=string, format=format): + got = date.strptime(string, format) + self.assertEqual(expected, got) + self.assertIs(type(got), date) + + def test_strptime_single_digit(self): + # bpo-34903: Check that single digit dates are allowed. + strptime = date.strptime + with self.assertRaises(ValueError): + # %y does require two digits. + newdate = strptime('01/02/3', '%d/%m/%y') + + d1 = date(2003, 2, 1) + d2 = date(2003, 1, 2) + d3 = date(2003, 1, 25) + inputs = [ + ('%d', '1/02/03', '%d/%m/%y', d1), + ('%m', '01/2/03', '%d/%m/%y', d1), + ('%j', '2/03', '%j/%y', d2), + ('%w', '6/04/03', '%w/%U/%y', d1), + # %u requires a single digit. + ('%W', '6/4/2003', '%u/%W/%Y', d1), + ('%V', '6/4/2003', '%u/%V/%G', d3), + ] + for reason, string, format, target in inputs: + reason = 'test single digit ' + reason + with self.subTest(reason=reason, + string=string, + format=format, + target=target): + newdate = strptime(string, format) + self.assertEqual(newdate, target, msg=reason) + + @warnings_helper.ignore_warnings(category=DeprecationWarning) + def test_strptime_leap_year(self): + # GH-70647: warns if parsing a format with a day and no year. + with self.assertRaises(ValueError): + # The existing behavior that GH-70647 seeks to change. + date.strptime('02-29', '%m-%d') + with self._assertNotWarns(DeprecationWarning): + date.strptime('20-03-14', '%y-%m-%d') + date.strptime('02-29,2024', '%m-%d,%Y') + class SubclassDate(date): sub_var = 1 @@ -1135,7 +1231,7 @@ def test_roundtrip(self): self.theclass.today()): # Verify dt -> string -> date identity. s = repr(dt) - self.assertTrue(s.startswith('datetime.')) + self.assertStartsWith(s, 'datetime.') s = s[9:] dt2 = eval(s) self.assertEqual(dt, dt2) @@ -1144,6 +1240,15 @@ def test_roundtrip(self): dt2 = self.theclass(dt.year, dt.month, dt.day) self.assertEqual(dt, dt2) + def test_repr_subclass(self): + """Subclasses should have bare names in the repr (gh-107773).""" + td = SubclassDate(1, 2, 3) + self.assertEqual(repr(td), "SubclassDate(1, 2, 3)") + td = SubclassDate(2014, 1, 1) + self.assertEqual(repr(td), "SubclassDate(2014, 1, 1)") + td = SubclassDate(2010, 10, day=10) + self.assertEqual(repr(td), "SubclassDate(2010, 10, 10)") + def test_ordinal_conversions(self): # Check some fixed values. for y, m, d, n in [(1, 1, 1, 1), # calendar origin @@ -1519,7 +1624,6 @@ def test_strftime(self): # bpo-41260: The parameter was named "fmt" in the pure python impl. t.strftime(format="%f") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_strftime_trailing_percent(self): # bpo-35066: Make sure trailing '%' doesn't cause datetime's strftime to # complain. Different libcs have different handling of trailing @@ -1620,7 +1724,6 @@ def test_pickling(self): self.assertEqual(orig, derived) self.assertEqual(orig.__reduce__(), orig.__reduce_ex__(2)) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_compat_unpickle(self): tests = [ b"cdatetime\ndate\n(S'\\x07\\xdf\\x0b\\x1b'\ntR.", @@ -1704,19 +1807,42 @@ def test_bool(self): self.assertTrue(self.theclass.min) self.assertTrue(self.theclass.max) + def check_strftime_y2k(self, specifier): + # Test that years less than 1000 are 0-padded; note that the beginning + # of an ISO 8601 year may fall in an ISO week of the year before, and + # therefore needs an offset of -1 when formatting with '%G'. + dataset = ( + (1, 0), + (49, -1), + (70, 0), + (99, 0), + (100, -1), + (999, 0), + (1000, 0), + (1970, 0), + ) + for year, g_offset in dataset: + with self.subTest(year=year, specifier=specifier): + d = self.theclass(year, 1, 1) + if specifier == 'G': + year += g_offset + if specifier == 'C': + expected = f"{year // 100:02d}" + else: + expected = f"{year:04d}" + if specifier == 'F': + expected += f"-01-01" + self.assertEqual(d.strftime(f"%{specifier}"), expected) + def test_strftime_y2k(self): - for y in (1, 49, 70, 99, 100, 999, 1000, 1970): - d = self.theclass(y, 1, 1) - # Issue 13305: For years < 1000, the value is not always - # padded to 4 digits across platforms. The C standard - # assumes year >= 1900, so it does not specify the number - # of digits. - if d.strftime("%Y") != '%04d' % y: - # Year 42 returns '42', not padded - self.assertEqual(d.strftime("%Y"), '%d' % y) - # '0042' is obtained anyway - if support.has_strftime_extensions: - self.assertEqual(d.strftime("%4Y"), '%04d' % y) + self.check_strftime_y2k('Y') + self.check_strftime_y2k('G') + + def test_strftime_y2k_c99(self): + # CPython requires C11; specifiers new in C99 must work. + # (Other implementations may want to disable this test.) + self.check_strftime_y2k('F') + self.check_strftime_y2k('C') def test_replace(self): cls = self.theclass @@ -1867,6 +1993,26 @@ def test_backdoor_resistance(self): # blow up because other fields are insane. self.theclass(base[:2] + bytes([ord_byte]) + base[3:]) + def test_valuerror_messages(self): + pattern = re.compile( + r"(year|month|day) must be in \d+\.\.\d+, not \d+" + ) + test_cases = [ + (2009, 13, 1), # Month out of range + (2009, 0, 1), # Month out of range + (10000, 12, 31), # Year out of range + (0, 12, 31), # Year out of range + ] + for case in test_cases: + with self.subTest(case): + with self.assertRaisesRegex(ValueError, pattern): + self.theclass(*case) + + # days out of range have their own error message, see issue 70647 + with self.assertRaises(ValueError) as msg: + self.theclass(2009, 1, 32) + self.assertIn(f"day 32 must be in range 1..31 for month 1 in year 2009", str(msg.exception)) + def test_fromisoformat(self): # Test that isoformat() is reversible base_dates = [ @@ -1950,6 +2096,7 @@ def test_fromisoformat_fails(self): '10000-W25-1', # Invalid year '2020-W25-0', # Invalid day-of-week '2020-W25-8', # Invalid day-of-week + '٢025-03-09' # Unicode characters '2009\ud80002\ud80028', # Separators are surrogate codepoints ] @@ -2077,7 +2224,7 @@ def test_roundtrip(self): self.theclass.now()): # Verify dt -> string -> datetime identity. s = repr(dt) - self.assertTrue(s.startswith('datetime.')) + self.assertStartsWith(s, 'datetime.') s = s[9:] dt2 = eval(s) self.assertEqual(dt, dt2) @@ -2407,7 +2554,6 @@ def test_pickling_subclass_datetime(self): self.assertEqual(orig, derived) self.assertTrue(isinstance(derived, SubclassDatetime)) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_compat_unpickle(self): tests = [ b'cdatetime\ndatetime\n(' @@ -2727,7 +2873,8 @@ def test_utcnow(self): def test_strptime(self): string = '2004-12-01 13:02:47.197' format = '%Y-%m-%d %H:%M:%S.%f' - expected = _strptime._strptime_datetime(self.theclass, string, format) + expected = _strptime._strptime_datetime_datetime(self.theclass, string, + format) got = self.theclass.strptime(string, format) self.assertEqual(expected, got) self.assertIs(type(expected), self.theclass) @@ -2741,8 +2888,8 @@ def test_strptime(self): ] for string, format in inputs: with self.subTest(string=string, format=format): - expected = _strptime._strptime_datetime(self.theclass, string, - format) + expected = _strptime._strptime_datetime_datetime(self.theclass, + string, format) got = self.theclass.strptime(string, format) self.assertEqual(expected, got) @@ -2864,8 +3011,7 @@ def test_more_strftime(self): self.assertEqual(t.strftime("%z"), "-0200" + z) self.assertEqual(t.strftime("%:z"), "-02:00:" + z) - @unittest.skip('TODO: RUSTPYTHON') - # UnicodeEncodeError: 'utf-8' codec can't encode character '\ud83d' in position 0: surrogates not allowed + @unittest.skip("TODO: RUSTPYTHON") def test_strftime_special(self): t = self.theclass(2004, 12, 31, 6, 22, 33, 47) s1 = t.strftime('%c') @@ -3119,6 +3265,28 @@ class DateTimeSubclass(self.theclass): self.assertEqual(res.year, 2013) self.assertEqual(res.fold, fold) + def test_valuerror_messages(self): + pattern = re.compile( + r"(year|month|day|hour|minute|second) must " + r"be in \d+\.\.\d+, not \d+" + ) + test_cases = [ + (2009, 4, 1, 12, 30, 90), # Second out of range + (2009, 4, 1, 12, 90, 45), # Minute out of range + (2009, 4, 1, 25, 30, 45), # Hour out of range + (2009, 13, 1, 24, 0, 0), # Month out of range + (9999, 12, 31, 24, 0, 0), # Year out of range + ] + for case in test_cases: + with self.subTest(case): + with self.assertRaisesRegex(ValueError, pattern): + self.theclass(*case) + + # days out of range have their own error message, see issue 70647 + with self.assertRaises(ValueError) as msg: + self.theclass(2009, 4, 32, 24, 0, 0) + self.assertIn(f"day 32 must be in range 1..30 for month 4 in year 2009", str(msg.exception)) + def test_fromisoformat_datetime(self): # Test that isoformat() is reversible base_dates = [ @@ -3360,6 +3528,9 @@ def test_fromisoformat_datetime_examples(self): ('2025-01-02T03:04:05,678+00:00:10', self.theclass(2025, 1, 2, 3, 4, 5, 678000, tzinfo=timezone(timedelta(seconds=10)))), + ('2025-01-02T24:00:00', self.theclass(2025, 1, 3, 0, 0, 0)), + ('2025-01-31T24:00:00', self.theclass(2025, 2, 1, 0, 0, 0)), + ('2025-12-31T24:00:00', self.theclass(2026, 1, 1, 0, 0, 0)) ] for input_str, expected in examples: @@ -3382,7 +3553,7 @@ def test_fromisoformat_fails_datetime(self): '2009-04-19T03:15:4500:00', # Bad time zone separator '2009-04-19T03:15:45.123456+24:30', # Invalid time zone offset '2009-04-19T03:15:45.123456-24:30', # Invalid negative offset - '2009-04-10ᛇᛇᛇᛇᛇ12:15', # Too many unicode separators + '2009-04-10ᛇᛇᛇᛇᛇ12:15', # Unicode chars '2009-04\ud80010T12:15', # Surrogate char in date '2009-04-10T12\ud80015', # Surrogate char in time '2009-04-19T1', # Incomplete hours @@ -3391,11 +3562,19 @@ def test_fromisoformat_fails_datetime(self): '2009-04-19T12:', # Ends with time separator '2009-04-19T12:30:', # Ends with time separator '2009-04-19T12:30:45.', # Ends with time separator - '2009-04-19T12:30:45.123456+', # Ends with timzone separator - '2009-04-19T12:30:45.123456-', # Ends with timzone separator + '2009-04-19T12:30:45.123456+', # Ends with timezone separator + '2009-04-19T12:30:45.123456-', # Ends with timezone separator '2009-04-19T12:30:45.123456-05:00a', # Extra text '2009-04-19T12:30:45.123-05:00a', # Extra text '2009-04-19T12:30:45-05:00a', # Extra text + '2009-04-19T24:00:00.000001', # Has non-zero microseconds on 24:00 + '2009-04-19T24:00:01.000000', # Has non-zero seconds on 24:00 + '2009-04-19T24:01:00.000000', # Has non-zero minutes on 24:00 + '2009-04-32T24:00:00.000000', # Day is invalid before wrapping due to 24:00 + '2009-13-01T24:00:00.000000', # Month is invalid before wrapping due to 24:00 + '9999-12-31T24:00:00.000000', # Year is invalid after wrapping due to 24:00 + '2009-04-19T12:30Z12:00', # Extra time zone info after Z + '2009-04-19T12:30:45:334034', # Invalid microsecond separator '2009-04-19T12:30:45.400 +02:30', # Space between ms and timezone (gh-130959) '2009-04-19T12:30:45.400 ', # Trailing space (gh-130959) '2009-04-19T12:30:45. 400', # Space before fraction (gh-130959) @@ -3406,6 +3585,29 @@ def test_fromisoformat_fails_datetime(self): with self.assertRaises(ValueError): self.theclass.fromisoformat(bad_str) + def test_fromisoformat_fails_datetime_valueerror(self): + pattern = re.compile( + r"(year|month|day|hour|minute|second) must " + r"be in \d+\.\.\d+, not \d+" + ) + bad_strs = [ + "2009-04-01T12:30:90", # Second out of range + "2009-04-01T12:90:45", # Minute out of range + "2009-04-01T25:30:45", # Hour out of range + "2009-13-01T24:00:00", # Month out of range + "9999-12-31T24:00:00", # Year out of range + ] + + for bad_str in bad_strs: + with self.subTest(bad_str=bad_str): + with self.assertRaisesRegex(ValueError, pattern): + self.theclass.fromisoformat(bad_str) + + # days out of range have their own error message, see issue 70647 + with self.assertRaises(ValueError) as msg: + self.theclass.fromisoformat("2009-04-32T24:00:00") + self.assertIn(f"day 32 must be in range 1..30 for month 4 in year 2009", str(msg.exception)) + def test_fromisoformat_fails_surrogate(self): # Test that when fromisoformat() fails with a surrogate character as # the separator, the error message contains the original string @@ -3432,6 +3634,15 @@ class DateTimeSubclass(self.theclass): self.assertEqual(dt, dt_rt) self.assertIsInstance(dt_rt, DateTimeSubclass) + def test_repr_subclass(self): + """Subclasses should have bare names in the repr (gh-107773).""" + td = SubclassDatetime(2014, 1, 1) + self.assertEqual(repr(td), "SubclassDatetime(2014, 1, 1, 0, 0)") + td = SubclassDatetime(2010, 10, day=10) + self.assertEqual(repr(td), "SubclassDatetime(2010, 10, 10, 0, 0)") + td = SubclassDatetime(2010, 10, 2, second=3) + self.assertEqual(repr(td), "SubclassDatetime(2010, 10, 2, 0, 0, 3)") + class TestSubclassDateTime(TestDateTime): theclass = SubclassDatetime @@ -3468,7 +3679,7 @@ def test_roundtrip(self): # Verify t -> string -> time identity. s = repr(t) - self.assertTrue(s.startswith('datetime.')) + self.assertStartsWith(s, 'datetime.') s = s[9:] t2 = eval(s) self.assertEqual(t, t2) @@ -3668,8 +3879,7 @@ def test_strftime(self): # gh-85432: The parameter was named "fmt" in the pure-Python impl. t.strftime(format="%f") - @unittest.skip('TODO: RUSTPYTHON') - # UnicodeEncodeError: 'utf-8' codec can't encode character '\ud83d' in position 0: surrogates not allowed + @unittest.skip("TODO: RUSTPYTHON") def test_strftime_special(self): t = self.theclass(1, 2, 3, 4) s1 = t.strftime('%I%p%Z') @@ -3744,6 +3954,19 @@ def test_repr(self): self.assertEqual(repr(self.theclass(23, 15, 0, 0)), "%s(23, 15)" % name) + def test_repr_subclass(self): + """Subclasses should have bare names in the repr (gh-107773).""" + td = SubclassTime(hour=1) + self.assertEqual(repr(td), "SubclassTime(1, 0)") + td = SubclassTime(hour=2, minute=30) + self.assertEqual(repr(td), "SubclassTime(2, 30)") + td = SubclassTime(hour=2, minute=30, second=11) + self.assertEqual(repr(td), "SubclassTime(2, 30, 11)") + td = SubclassTime(minute=30, second=11, fold=0) + self.assertEqual(repr(td), "SubclassTime(0, 30, 11)") + td = SubclassTime(minute=30, second=11, fold=1) + self.assertEqual(repr(td), "SubclassTime(0, 30, 11, fold=1)") + def test_resolution_info(self): self.assertIsInstance(self.theclass.min, self.theclass) self.assertIsInstance(self.theclass.max, self.theclass) @@ -3768,7 +3991,6 @@ def test_pickling_subclass_time(self): self.assertEqual(orig, derived) self.assertTrue(isinstance(derived, SubclassTime)) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_compat_unpickle(self): tests = [ (b"cdatetime\ntime\n(S'\\x14;\\x10\\x00\\x10\\x00'\ntR.", @@ -3791,6 +4013,78 @@ def test_compat_unpickle(self): derived = loads(data, encoding='latin1') self.assertEqual(derived, expected) + def test_strptime(self): + # bpo-34482: Check that surrogates are handled properly. + inputs = [ + (self.theclass(13, 2, 47, 197000), '13:02:47.197', '%H:%M:%S.%f'), + (self.theclass(13, 2, 47, 197000), '13:02\ud80047.197', '%H:%M\ud800%S.%f'), + (self.theclass(13, 2, 47, 197000), '13\ud80002:47.197', '%H\ud800%M:%S.%f'), + ] + for expected, string, format in inputs: + with self.subTest(string=string, format=format): + got = self.theclass.strptime(string, format) + self.assertEqual(expected, got) + self.assertIs(type(got), self.theclass) + + def test_strptime_tz(self): + strptime = self.theclass.strptime + self.assertEqual(strptime("+0002", "%z").utcoffset(), 2 * MINUTE) + self.assertEqual(strptime("-0002", "%z").utcoffset(), -2 * MINUTE) + self.assertEqual( + strptime("-00:02:01.000003", "%z").utcoffset(), + -timedelta(minutes=2, seconds=1, microseconds=3) + ) + # Only local timezone and UTC are supported + for tzseconds, tzname in ((0, 'UTC'), (0, 'GMT'), + (-_time.timezone, _time.tzname[0])): + if tzseconds < 0: + sign = '-' + seconds = -tzseconds + else: + sign ='+' + seconds = tzseconds + hours, minutes = divmod(seconds//60, 60) + tstr = "{}{:02d}{:02d} {}".format(sign, hours, minutes, tzname) + with self.subTest(tstr=tstr): + t = strptime(tstr, "%z %Z") + self.assertEqual(t.utcoffset(), timedelta(seconds=tzseconds)) + self.assertEqual(t.tzname(), tzname) + self.assertIs(type(t), self.theclass) + + # Can produce inconsistent time + tstr, fmt = "+1234 UTC", "%z %Z" + t = strptime(tstr, fmt) + self.assertEqual(t.utcoffset(), 12 * HOUR + 34 * MINUTE) + self.assertEqual(t.tzname(), 'UTC') + # yet will roundtrip + self.assertEqual(t.strftime(fmt), tstr) + + # Produce naive time if no %z is provided + self.assertEqual(strptime("UTC", "%Z").tzinfo, None) + + def test_strptime_errors(self): + for tzstr in ("-2400", "-000", "z"): + with self.assertRaises(ValueError): + self.theclass.strptime(tzstr, "%z") + + def test_strptime_single_digit(self): + # bpo-34903: Check that single digit times are allowed. + t = self.theclass(4, 5, 6) + inputs = [ + ('%H', '4:05:06', '%H:%M:%S', t), + ('%M', '04:5:06', '%H:%M:%S', t), + ('%S', '04:05:6', '%H:%M:%S', t), + ('%I', '4am:05:06', '%I%p:%M:%S', t), + ] + for reason, string, format, target in inputs: + reason = 'test single digit ' + reason + with self.subTest(reason=reason, + string=string, + format=format, + target=target): + newdate = self.theclass.strptime(string, format) + self.assertEqual(newdate, target, msg=reason) + def test_bool(self): # time is always True. cls = self.theclass @@ -4066,8 +4360,7 @@ def test_empty(self): self.assertEqual(t.microsecond, 0) self.assertIsNone(t.tzinfo) - @unittest.skip('TODO: RUSTPYTHON') - # UnicodeEncodeError: 'utf-8' codec can't encode character '\ud800' in position 0: surrogates not allowed + @unittest.skip("TODO: RUSTPYTHON") def test_zones(self): est = FixedOffset(-300, "EST", 1) utc = FixedOffset(0, "UTC", -2) @@ -4186,7 +4479,6 @@ def test_pickling(self): self.assertEqual(derived.tzname(), 'cookie') self.assertEqual(orig.__reduce__(), orig.__reduce_ex__(2)) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_compat_unpickle(self): tests = [ b"cdatetime\ntime\n(S'\\x05\\x06\\x07\\x01\\xe2@'\n" @@ -4316,6 +4608,21 @@ def utcoffset(self, t): t2 = t2.replace(tzinfo=Varies()) self.assertTrue(t1 < t2) # t1's offset counter still going up + def test_valuerror_messages(self): + pattern = re.compile( + r"(hour|minute|second|microsecond) must be in \d+\.\.\d+, not \d+" + ) + test_cases = [ + (12, 30, 90, 9999991), # Microsecond out of range + (12, 30, 90, 000000), # Second out of range + (25, 30, 45, 000000), # Hour out of range + (12, 90, 45, 000000), # Minute out of range + ] + for case in test_cases: + with self.subTest(case): + with self.assertRaisesRegex(ValueError, pattern): + self.theclass(*case) + def test_fromisoformat(self): time_examples = [ (0, 0, 0, 0), @@ -4365,7 +4672,7 @@ def test_fromisoformat_timezone(self): with self.subTest(tstr=tstr): t_rt = self.theclass.fromisoformat(tstr) - assert t == t_rt, t_rt + assert t == t_rt def test_fromisoformat_timespecs(self): time_bases = [ @@ -4424,6 +4731,7 @@ def test_fromisoformat_time_examples(self): ('00:00:00.000', self.theclass(0, 0)), ('000000.000000', self.theclass(0, 0)), ('00:00:00.000000', self.theclass(0, 0)), + ('00:00:00,100000', self.theclass(0, 0, 0, 100000)), ('1200', self.theclass(12, 0)), ('12:00', self.theclass(12, 0)), ('120000', self.theclass(12, 0)), @@ -4489,6 +4797,10 @@ def test_fromisoformat_fails(self): '12:30:45.123456-', # Extra at end of microsecond time '12:30:45.123456+', # Extra at end of microsecond time '12:30:45.123456+12:00:30a', # Extra at end of full time + '12.5', # Decimal mark at end of hour + '12:30,5', # Decimal mark at end of minute + '12:30:45.123456Z12:00', # Extra time zone info after Z + '12:30:45:334034', # Invalid microsecond separator '12:30:45.400 +02:30', # Space between ms and timezone (gh-130959) '12:30:45.400 ', # Trailing space (gh-130959) '12:30:45. 400', # Space before fraction (gh-130959) @@ -4652,7 +4964,6 @@ def test_pickling(self): self.assertEqual(derived.tzname(), 'cookie') self.assertEqual(orig.__reduce__(), orig.__reduce_ex__(2)) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_compat_unpickle(self): tests = [ b'cdatetime\ndatetime\n' @@ -5829,21 +6140,21 @@ def test_vilnius_1941_fromutc(self): gdt = datetime(1941, 6, 23, 20, 59, 59, tzinfo=timezone.utc) ldt = gdt.astimezone(Vilnius) - self.assertEqual(ldt.strftime("%c %Z%z"), + self.assertEqual(ldt.strftime("%a %b %d %H:%M:%S %Y %Z%z"), 'Mon Jun 23 23:59:59 1941 MSK+0300') self.assertEqual(ldt.fold, 0) self.assertFalse(ldt.dst()) gdt = datetime(1941, 6, 23, 21, tzinfo=timezone.utc) ldt = gdt.astimezone(Vilnius) - self.assertEqual(ldt.strftime("%c %Z%z"), + self.assertEqual(ldt.strftime("%a %b %d %H:%M:%S %Y %Z%z"), 'Mon Jun 23 23:00:00 1941 CEST+0200') self.assertEqual(ldt.fold, 1) self.assertTrue(ldt.dst()) gdt = datetime(1941, 6, 23, 22, tzinfo=timezone.utc) ldt = gdt.astimezone(Vilnius) - self.assertEqual(ldt.strftime("%c %Z%z"), + self.assertEqual(ldt.strftime("%a %b %d %H:%M:%S %Y %Z%z"), 'Tue Jun 24 00:00:00 1941 CEST+0200') self.assertEqual(ldt.fold, 0) self.assertTrue(ldt.dst()) @@ -5853,22 +6164,22 @@ def test_vilnius_1941_toutc(self): ldt = datetime(1941, 6, 23, 22, 59, 59, tzinfo=Vilnius) gdt = ldt.astimezone(timezone.utc) - self.assertEqual(gdt.strftime("%c %Z"), + self.assertEqual(gdt.strftime("%a %b %d %H:%M:%S %Y %Z"), 'Mon Jun 23 19:59:59 1941 UTC') ldt = datetime(1941, 6, 23, 23, 59, 59, tzinfo=Vilnius) gdt = ldt.astimezone(timezone.utc) - self.assertEqual(gdt.strftime("%c %Z"), + self.assertEqual(gdt.strftime("%a %b %d %H:%M:%S %Y %Z"), 'Mon Jun 23 20:59:59 1941 UTC') ldt = datetime(1941, 6, 23, 23, 59, 59, tzinfo=Vilnius, fold=1) gdt = ldt.astimezone(timezone.utc) - self.assertEqual(gdt.strftime("%c %Z"), + self.assertEqual(gdt.strftime("%a %b %d %H:%M:%S %Y %Z"), 'Mon Jun 23 21:59:59 1941 UTC') ldt = datetime(1941, 6, 24, 0, tzinfo=Vilnius) gdt = ldt.astimezone(timezone.utc) - self.assertEqual(gdt.strftime("%c %Z"), + self.assertEqual(gdt.strftime("%a %b %d %H:%M:%S %Y %Z"), 'Mon Jun 23 22:00:00 1941 UTC') def test_constructors(self): @@ -6394,6 +6705,17 @@ def test_gaps(self): ldt = tz.fromutc(udt.replace(tzinfo=tz)) self.assertEqual(ldt.fold, 0) + @classmethod + @contextlib.contextmanager + def _change_tz(cls, new_tzinfo): + try: + with os_helper.EnvironmentVarGuard() as env: + env["TZ"] = new_tzinfo + _time.tzset() + yield + finally: + _time.tzset() + @unittest.skipUnless( hasattr(_time, "tzset"), "time module has no attribute tzset" ) @@ -6408,10 +6730,7 @@ def test_system_transitions(self): self.zonename.startswith('right/')): self.skipTest("Skipping %s" % self.zonename) tz = self.tz - TZ = os.environ.get('TZ') - os.environ['TZ'] = self.zonename - try: - _time.tzset() + with self._change_tz(self.zonename): for udt, shift in tz.transitions(): if udt.year >= 2037: # System support for times around the end of 32-bit time_t @@ -6419,7 +6738,7 @@ def test_system_transitions(self): break s0 = (udt - datetime(1970, 1, 1)) // SEC ss = shift // SEC # shift seconds - for x in [-40 * 3600, -20*3600, -1, 0, + for x in [-40 * 3600, -20 * 3600, -1, 0, ss - 1, ss + 20 * 3600, ss + 40 * 3600]: s = s0 + x sdt = datetime.fromtimestamp(s) @@ -6438,12 +6757,6 @@ def test_system_transitions(self): utc0 = dt.astimezone(timezone.utc) utc1 = dt.replace(fold=1).astimezone(timezone.utc) self.assertEqual(utc0, utc1 + timedelta(0, ss)) - finally: - if TZ is None: - del os.environ['TZ'] - else: - os.environ['TZ'] = TZ - _time.tzset() class ZoneInfoCompleteTest(unittest.TestSuite): @@ -6939,9 +7252,6 @@ def pickle_fake_date(datetime_) -> Type[FakeDate]: """) script_helper.assert_python_ok('-c', script) - # TODO: RUSTPYTHON - # AssertionError: Process return code is 1 - @unittest.expectedFailure def test_update_type_cache(self): # gh-120782 script = textwrap.dedent(""" @@ -6974,6 +7284,34 @@ def test_update_type_cache(self): """) script_helper.assert_python_ok('-c', script) + def test_concurrent_initialization_subinterpreter(self): + # gh-136421: Concurrent initialization of _datetime across multiple + # interpreters wasn't thread-safe due to its static types. + + # Run in a subprocess to ensure we get a clean version of _datetime + script = """if True: + from concurrent.futures import InterpreterPoolExecutor + + def func(): + import _datetime + print('a', end='') + + with InterpreterPoolExecutor() as executor: + for _ in range(8): + executor.submit(func) + """ + rc, out, err = script_helper.assert_python_ok("-c", script) + self.assertEqual(rc, 0) + self.assertEqual(out, b"a" * 8) + self.assertEqual(err, b"") + + # Now test against concurrent reinitialization + script = "import _datetime\n" + script + rc, out, err = script_helper.assert_python_ok("-c", script) + self.assertEqual(rc, 0) + self.assertEqual(out, b"a" * 8) + self.assertEqual(err, b"") + def load_tests(loader, standard_tests, pattern): standard_tests.addTest(ZoneInfoCompleteTest()) diff --git a/Lib/test/dis_module.py b/Lib/test/dis_module.py new file mode 100644 index 00000000000..afbf600fdee --- /dev/null +++ b/Lib/test/dis_module.py @@ -0,0 +1,5 @@ + +# A simple module for testing the dis module. + +def f(): pass +def g(): pass diff --git a/Lib/test/libregrtest/save_env.py b/Lib/test/libregrtest/save_env.py index 0ec12d7c475..eeb1f17d702 100644 --- a/Lib/test/libregrtest/save_env.py +++ b/Lib/test/libregrtest/save_env.py @@ -240,7 +240,9 @@ def get_multiprocessing_process__dangling(self): # Unjoined process objects can survive after process exits multiprocessing_process._cleanup() # This copies the weakrefs without making any strong reference - return multiprocessing_process._dangling.copy() + # TODO: RUSTPYTHON - filter out dead processes since gc doesn't clean WeakSet. Revert this line when we have a GC + # return multiprocessing_process._dangling.copy() + return {p for p in multiprocessing_process._dangling if p.is_alive()} def restore_multiprocessing_process__dangling(self, saved): multiprocessing_process = self.get_module('multiprocessing.process') multiprocessing_process._dangling.clear() diff --git a/Lib/test/list_tests.py b/Lib/test/list_tests.py index 65dfa41b26e..e76f79c274e 100644 --- a/Lib/test/list_tests.py +++ b/Lib/test/list_tests.py @@ -6,7 +6,8 @@ from functools import cmp_to_key from test import seq_tests -from test.support import ALWAYS_EQ, NEVER_EQ, get_c_recursion_limit +from test.support import ALWAYS_EQ, NEVER_EQ +from test.support import skip_emscripten_stack_overflow, skip_wasi_stack_overflow class CommonTest(seq_tests.CommonTest): @@ -59,9 +60,11 @@ def test_repr(self): self.assertEqual(str(a2), "[0, 1, 2, [...], 3]") self.assertEqual(repr(a2), "[0, 1, 2, [...], 3]") + @skip_wasi_stack_overflow() + @skip_emscripten_stack_overflow() def test_repr_deep(self): a = self.type2test([]) - for i in range(get_c_recursion_limit() + 1): + for i in range(200_000): a = self.type2test([a]) self.assertRaises(RecursionError, repr, a) diff --git a/Lib/test/lock_tests.py b/Lib/test/lock_tests.py index 4031bfaeb64..8c7a4f76563 100644 --- a/Lib/test/lock_tests.py +++ b/Lib/test/lock_tests.py @@ -162,7 +162,7 @@ def f(): self.assertFalse(result[0]) lock.release() - @unittest.skip('TODO: RUSTPYTHON; sometimes hangs') + @unittest.skip("TODO: RUSTPYTHON; sometimes hangs") def test_acquire_contended(self): lock = self.locktype() lock.acquire() @@ -210,7 +210,7 @@ def with_lock(err=None): with Bunch(f, 1): pass - @unittest.skip('TODO: RUSTPYTHON; sometimes hangs') + @unittest.skip("TODO: RUSTPYTHON; sometimes hangs") def test_thread_leak(self): # The lock shouldn't leak a Thread instance when used from a foreign # (non-threading) thread. @@ -334,6 +334,26 @@ class RLockTests(BaseLockTests): """ Tests for recursive locks. """ + def test_repr_count(self): + # see gh-134322: check that count values are correct: + # when a rlock is just created, + # in a second thread when rlock is acquired in the main thread. + lock = self.locktype() + self.assertIn("count=0", repr(lock)) + self.assertIn("= 1: + self.fail("changing dict size during iteration doesn't raise Error") + count += 1 except RuntimeError: pass - else: - self.fail("changing dict size during iteration doesn't raise Error") def test_repr(self): d = self._empty_mapping() @@ -620,9 +622,12 @@ def __repr__(self): d = self._full_mapping({1: BadRepr()}) self.assertRaises(Exc, repr, d) + @support.skip_wasi_stack_overflow() + @support.skip_emscripten_stack_overflow() + @support.skip_if_sanitizer("requires deep stack", ub=True) def test_repr_deep(self): d = self._empty_mapping() - for i in range(sys.getrecursionlimit() + 100): + for i in range(support.exceeds_recursion_limit()): d0 = d d = self._empty_mapping() d[1] = d0 diff --git a/Lib/test/multibytecodec_support.py b/Lib/test/multibytecodec_support.py new file mode 100644 index 00000000000..6b4c57d0b4b --- /dev/null +++ b/Lib/test/multibytecodec_support.py @@ -0,0 +1,400 @@ +# +# multibytecodec_support.py +# Common Unittest Routines for CJK codecs +# + +import codecs +import os +import re +import sys +import unittest +from http.client import HTTPException +from test import support +from io import BytesIO + +class TestBase: + encoding = '' # codec name + codec = None # codec tuple (with 4 elements) + tstring = None # must set. 2 strings to test StreamReader + + codectests = None # must set. codec test tuple + roundtriptest = 1 # set if roundtrip is possible with unicode + has_iso10646 = 0 # set if this encoding contains whole iso10646 map + xmlcharnametest = None # string to test xmlcharrefreplace + unmappedunicode = '\udeee' # a unicode code point that is not mapped. + + def setUp(self): + if self.codec is None: + self.codec = codecs.lookup(self.encoding) + self.encode = self.codec.encode + self.decode = self.codec.decode + self.reader = self.codec.streamreader + self.writer = self.codec.streamwriter + self.incrementalencoder = self.codec.incrementalencoder + self.incrementaldecoder = self.codec.incrementaldecoder + + def test_chunkcoding(self): + tstring_lines = [] + for b in self.tstring: + lines = b.split(b"\n") + last = lines.pop() + assert last == b"" + lines = [line + b"\n" for line in lines] + tstring_lines.append(lines) + for native, utf8 in zip(*tstring_lines): + u = self.decode(native)[0] + self.assertEqual(u, utf8.decode('utf-8')) + if self.roundtriptest: + self.assertEqual(native, self.encode(u)[0]) + + def test_errorhandle(self): + for source, scheme, expected in self.codectests: + if isinstance(source, bytes): + func = self.decode + else: + func = self.encode + if expected: + result = func(source, scheme)[0] + if func is self.decode: + self.assertTrue(type(result) is str, type(result)) + self.assertEqual(result, expected, + '%a.decode(%r, %r)=%a != %a' + % (source, self.encoding, scheme, result, + expected)) + else: + self.assertTrue(type(result) is bytes, type(result)) + self.assertEqual(result, expected, + '%a.encode(%r, %r)=%a != %a' + % (source, self.encoding, scheme, result, + expected)) + else: + self.assertRaises(UnicodeError, func, source, scheme) + + def test_xmlcharrefreplace(self): + if self.has_iso10646: + self.skipTest('encoding contains full ISO 10646 map') + + s = "\u0b13\u0b23\u0b60 nd eggs" + self.assertEqual( + self.encode(s, "xmlcharrefreplace")[0], + b"ଓଣୠ nd eggs" + ) + + def test_customreplace_encode(self): + if self.has_iso10646: + self.skipTest('encoding contains full ISO 10646 map') + + from html.entities import codepoint2name + + def xmlcharnamereplace(exc): + if not isinstance(exc, UnicodeEncodeError): + raise TypeError("don't know how to handle %r" % exc) + l = [] + for c in exc.object[exc.start:exc.end]: + if ord(c) in codepoint2name: + l.append("&%s;" % codepoint2name[ord(c)]) + else: + l.append("&#%d;" % ord(c)) + return ("".join(l), exc.end) + + codecs.register_error("test.xmlcharnamereplace", xmlcharnamereplace) + + if self.xmlcharnametest: + sin, sout = self.xmlcharnametest + else: + sin = "\xab\u211c\xbb = \u2329\u1234\u232a" + sout = b"«ℜ» = ⟨ሴ⟩" + self.assertEqual(self.encode(sin, + "test.xmlcharnamereplace")[0], sout) + + def test_callback_returns_bytes(self): + def myreplace(exc): + return (b"1234", exc.end) + codecs.register_error("test.cjktest", myreplace) + enc = self.encode("abc" + self.unmappedunicode + "def", "test.cjktest")[0] + self.assertEqual(enc, b"abc1234def") + + def test_callback_wrong_objects(self): + def myreplace(exc): + return (ret, exc.end) + codecs.register_error("test.cjktest", myreplace) + + for ret in ([1, 2, 3], [], None, object()): + self.assertRaises(TypeError, self.encode, self.unmappedunicode, + 'test.cjktest') + + def test_callback_long_index(self): + def myreplace(exc): + return ('x', int(exc.end)) + codecs.register_error("test.cjktest", myreplace) + self.assertEqual(self.encode('abcd' + self.unmappedunicode + 'efgh', + 'test.cjktest'), (b'abcdxefgh', 9)) + + def myreplace(exc): + return ('x', sys.maxsize + 1) + codecs.register_error("test.cjktest", myreplace) + self.assertRaises(IndexError, self.encode, self.unmappedunicode, + 'test.cjktest') + + def test_callback_None_index(self): + def myreplace(exc): + return ('x', None) + codecs.register_error("test.cjktest", myreplace) + self.assertRaises(TypeError, self.encode, self.unmappedunicode, + 'test.cjktest') + + def test_callback_backward_index(self): + def myreplace(exc): + if myreplace.limit > 0: + myreplace.limit -= 1 + return ('REPLACED', 0) + else: + return ('TERMINAL', exc.end) + myreplace.limit = 3 + codecs.register_error("test.cjktest", myreplace) + self.assertEqual(self.encode('abcd' + self.unmappedunicode + 'efgh', + 'test.cjktest'), + (b'abcdREPLACEDabcdREPLACEDabcdREPLACEDabcdTERMINALefgh', 9)) + + def test_callback_forward_index(self): + def myreplace(exc): + return ('REPLACED', exc.end + 2) + codecs.register_error("test.cjktest", myreplace) + self.assertEqual(self.encode('abcd' + self.unmappedunicode + 'efgh', + 'test.cjktest'), (b'abcdREPLACEDgh', 9)) + + def test_callback_index_outofbound(self): + def myreplace(exc): + return ('TERM', 100) + codecs.register_error("test.cjktest", myreplace) + self.assertRaises(IndexError, self.encode, self.unmappedunicode, + 'test.cjktest') + + def test_incrementalencoder(self): + UTF8Reader = codecs.getreader('utf-8') + for sizehint in [None] + list(range(1, 33)) + \ + [64, 128, 256, 512, 1024]: + istream = UTF8Reader(BytesIO(self.tstring[1])) + ostream = BytesIO() + encoder = self.incrementalencoder() + while 1: + if sizehint is not None: + data = istream.read(sizehint) + else: + data = istream.read() + + if not data: + break + e = encoder.encode(data) + ostream.write(e) + + self.assertEqual(ostream.getvalue(), self.tstring[0]) + + def test_incrementaldecoder(self): + UTF8Writer = codecs.getwriter('utf-8') + for sizehint in [None, -1] + list(range(1, 33)) + \ + [64, 128, 256, 512, 1024]: + istream = BytesIO(self.tstring[0]) + ostream = UTF8Writer(BytesIO()) + decoder = self.incrementaldecoder() + while 1: + data = istream.read(sizehint) + if not data: + break + else: + u = decoder.decode(data) + ostream.write(u) + + self.assertEqual(ostream.getvalue(), self.tstring[1]) + + def test_incrementalencoder_error_callback(self): + inv = self.unmappedunicode + + e = self.incrementalencoder() + self.assertRaises(UnicodeEncodeError, e.encode, inv, True) + + e.errors = 'ignore' + self.assertEqual(e.encode(inv, True), b'') + + e.reset() + def tempreplace(exc): + return ('called', exc.end) + codecs.register_error('test.incremental_error_callback', tempreplace) + e.errors = 'test.incremental_error_callback' + self.assertEqual(e.encode(inv, True), b'called') + + # again + e.errors = 'ignore' + self.assertEqual(e.encode(inv, True), b'') + + def test_streamreader(self): + UTF8Writer = codecs.getwriter('utf-8') + for name in ["read", "readline", "readlines"]: + for sizehint in [None, -1] + list(range(1, 33)) + \ + [64, 128, 256, 512, 1024]: + istream = self.reader(BytesIO(self.tstring[0])) + ostream = UTF8Writer(BytesIO()) + func = getattr(istream, name) + while 1: + data = func(sizehint) + if not data: + break + if name == "readlines": + ostream.writelines(data) + else: + ostream.write(data) + + self.assertEqual(ostream.getvalue(), self.tstring[1]) + + def test_streamwriter(self): + readfuncs = ('read', 'readline', 'readlines') + UTF8Reader = codecs.getreader('utf-8') + for name in readfuncs: + for sizehint in [None] + list(range(1, 33)) + \ + [64, 128, 256, 512, 1024]: + istream = UTF8Reader(BytesIO(self.tstring[1])) + ostream = self.writer(BytesIO()) + func = getattr(istream, name) + while 1: + if sizehint is not None: + data = func(sizehint) + else: + data = func() + + if not data: + break + if name == "readlines": + ostream.writelines(data) + else: + ostream.write(data) + + self.assertEqual(ostream.getvalue(), self.tstring[0]) + + def test_streamwriter_reset_no_pending(self): + # Issue #23247: Calling reset() on a fresh StreamWriter instance + # (without pending data) must not crash + stream = BytesIO() + writer = self.writer(stream) + writer.reset() + + def test_incrementalencoder_del_segfault(self): + e = self.incrementalencoder() + with self.assertRaises(AttributeError): + del e.errors + + def test_null_terminator(self): + # see gh-101828 + text = "フルーツ" + try: + text.encode(self.encoding) + except UnicodeEncodeError: + text = "Python is cool" + encode_w_null = (text + "\0").encode(self.encoding) + encode_plus_null = text.encode(self.encoding) + "\0".encode(self.encoding) + self.assertTrue(encode_w_null.endswith(b'\x00')) + self.assertEqual(encode_w_null, encode_plus_null) + + encode_w_null_2 = (text + "\0" + text + "\0").encode(self.encoding) + encode_plus_null_2 = encode_plus_null + encode_plus_null + self.assertEqual(encode_w_null_2.count(b'\x00'), 2) + self.assertEqual(encode_w_null_2, encode_plus_null_2) + + +class TestBase_Mapping(unittest.TestCase): + pass_enctest = [] + pass_dectest = [] + supmaps = [] + codectests = [] + + def setUp(self): + try: + self.open_mapping_file().close() # test it to report the error early + except (OSError, HTTPException): + self.skipTest("Could not retrieve "+self.mapfileurl) + + def open_mapping_file(self): + return support.open_urlresource(self.mapfileurl, encoding="utf-8") + + def test_mapping_file(self): + if self.mapfileurl.endswith('.xml'): + self._test_mapping_file_ucm() + else: + self._test_mapping_file_plain() + + def _test_mapping_file_plain(self): + def unichrs(s): + return ''.join(chr(int(x, 16)) for x in s.split('+')) + + urt_wa = {} + + with self.open_mapping_file() as f: + for line in f: + if not line: + break + data = line.split('#')[0].split() + if len(data) != 2: + continue + + if data[0][:2] != '0x': + self.fail(f"Invalid line: {line!r}") + csetch = bytes.fromhex(data[0][2:]) + if len(csetch) == 1 and 0x80 <= csetch[0]: + continue + + unich = unichrs(data[1]) + if ord(unich) == 0xfffd or unich in urt_wa: + continue + urt_wa[unich] = csetch + + self._testpoint(csetch, unich) + + def _test_mapping_file_ucm(self): + with self.open_mapping_file() as f: + ucmdata = f.read() + uc = re.findall('', ucmdata) + for uni, coded in uc: + unich = chr(int(uni, 16)) + codech = bytes.fromhex(coded) + self._testpoint(codech, unich) + + def test_mapping_supplemental(self): + for mapping in self.supmaps: + self._testpoint(*mapping) + + def _testpoint(self, csetch, unich): + if (csetch, unich) not in self.pass_enctest: + self.assertEqual(unich.encode(self.encoding), csetch) + if (csetch, unich) not in self.pass_dectest: + self.assertEqual(str(csetch, self.encoding), unich) + + def test_errorhandle(self): + for source, scheme, expected in self.codectests: + if isinstance(source, bytes): + func = source.decode + else: + func = source.encode + if expected: + if isinstance(source, bytes): + result = func(self.encoding, scheme) + self.assertTrue(type(result) is str, type(result)) + self.assertEqual(result, expected, + '%a.decode(%r, %r)=%a != %a' + % (source, self.encoding, scheme, result, + expected)) + else: + result = func(self.encoding, scheme) + self.assertTrue(type(result) is bytes, type(result)) + self.assertEqual(result, expected, + '%a.encode(%r, %r)=%a != %a' + % (source, self.encoding, scheme, result, + expected)) + else: + self.assertRaises(UnicodeError, func, self.encoding, scheme) + +def load_teststring(name): + dir = os.path.join(os.path.dirname(__file__), 'cjkencodings') + with open(os.path.join(dir, name + '.txt'), 'rb') as f: + encoded = f.read() + with open(os.path.join(dir, name + '-utf8.txt'), 'rb') as f: + utf8 = f.read() + return encoded, utf8 diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index c0d4c8f43b9..9a3a26a8400 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -1012,6 +1012,26 @@ def test_constants(self): self.assertIs(self.loads(b'I01\n.'), True) self.assertIs(self.loads(b'I00\n.'), False) + def test_zero_padded_integers(self): + self.assertEqual(self.loads(b'I010\n.'), 10) + self.assertEqual(self.loads(b'I-010\n.'), -10) + self.assertEqual(self.loads(b'I0010\n.'), 10) + self.assertEqual(self.loads(b'I-0010\n.'), -10) + self.assertEqual(self.loads(b'L010\n.'), 10) + self.assertEqual(self.loads(b'L-010\n.'), -10) + self.assertEqual(self.loads(b'L0010\n.'), 10) + self.assertEqual(self.loads(b'L-0010\n.'), -10) + self.assertEqual(self.loads(b'L010L\n.'), 10) + self.assertEqual(self.loads(b'L-010L\n.'), -10) + + def test_nondecimal_integers(self): + self.assertRaises(ValueError, self.loads, b'I0b10\n.') + self.assertRaises(ValueError, self.loads, b'I0o10\n.') + self.assertRaises(ValueError, self.loads, b'I0x10\n.') + self.assertRaises(ValueError, self.loads, b'L0b10L\n.') + self.assertRaises(ValueError, self.loads, b'L0o10L\n.') + self.assertRaises(ValueError, self.loads, b'L0x10L\n.') + def test_empty_bytestring(self): # issue 11286 empty = self.loads(b'\x80\x03U\x00q\x00.', encoding='koi8-r') @@ -1234,24 +1254,37 @@ def test_find_class(self): self.assertIs(unpickler.find_class('os.path', 'join'), os.path.join) self.assertIs(unpickler4.find_class('builtins', 'str.upper'), str.upper) - with self.assertRaises(AttributeError): + with self.assertRaisesRegex(AttributeError, + r"module 'builtins' has no attribute 'str\.upper'"): unpickler.find_class('builtins', 'str.upper') - with self.assertRaises(AttributeError): + with self.assertRaisesRegex(AttributeError, + "module 'math' has no attribute 'spam'"): unpickler.find_class('math', 'spam') - with self.assertRaises(AttributeError): + with self.assertRaisesRegex(AttributeError, + "module 'math' has no attribute 'spam'"): unpickler4.find_class('math', 'spam') - with self.assertRaises(AttributeError): + with self.assertRaisesRegex(AttributeError, + r"module 'math' has no attribute 'log\.spam'"): unpickler.find_class('math', 'log.spam') - with self.assertRaises(AttributeError): + with self.assertRaisesRegex(AttributeError, + r"Can't resolve path 'log\.spam' on module 'math'") as cm: unpickler4.find_class('math', 'log.spam') - with self.assertRaises(AttributeError): + self.assertEqual(str(cm.exception.__context__), + "'builtin_function_or_method' object has no attribute 'spam'") + with self.assertRaisesRegex(AttributeError, + r"module 'math' has no attribute 'log\.\.spam'"): unpickler.find_class('math', 'log..spam') - with self.assertRaises(AttributeError): + with self.assertRaisesRegex(AttributeError, + r"Can't resolve path 'log\.\.spam' on module 'math'") as cm: unpickler4.find_class('math', 'log..spam') - with self.assertRaises(AttributeError): + self.assertEqual(str(cm.exception.__context__), + "'builtin_function_or_method' object has no attribute ''") + with self.assertRaisesRegex(AttributeError, + "module 'math' has no attribute ''"): unpickler.find_class('math', '') - with self.assertRaises(AttributeError): + with self.assertRaisesRegex(AttributeError, + "module 'math' has no attribute ''"): unpickler4.find_class('math', '') self.assertRaises(ModuleNotFoundError, unpickler.find_class, 'spam', 'log') self.assertRaises(ValueError, unpickler.find_class, '', 'log') @@ -1637,48 +1670,77 @@ def test_bad_reduce_result(self): obj = REX([print, ()]) for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(pickle.PicklingError): + with self.assertRaises(pickle.PicklingError) as cm: self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + '__reduce__ must return a string or tuple, not list') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) obj = REX((print,)) for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(pickle.PicklingError): + with self.assertRaises(pickle.PicklingError) as cm: self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'tuple returned by __reduce__ must contain 2 through 6 elements') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) obj = REX((print, (), None, None, None, None, None)) for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(pickle.PicklingError): + with self.assertRaises(pickle.PicklingError) as cm: self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'tuple returned by __reduce__ must contain 2 through 6 elements') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) def test_bad_reconstructor(self): obj = REX((42, ())) for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(pickle.PicklingError): + with self.assertRaises(pickle.PicklingError) as cm: self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'first item of the tuple returned by __reduce__ ' + 'must be callable, not int') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) def test_unpickleable_reconstructor(self): obj = REX((UnpickleableCallable(), ())) for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(CustomError): + with self.assertRaises(CustomError) as cm: self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX reconstructor', + 'when serializing test.pickletester.REX object']) def test_bad_reconstructor_args(self): obj = REX((print, [])) for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(pickle.PicklingError): + with self.assertRaises(pickle.PicklingError) as cm: self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'second item of the tuple returned by __reduce__ ' + 'must be a tuple, not list') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) def test_unpickleable_reconstructor_args(self): obj = REX((print, (1, 2, UNPICKLEABLE))) for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(CustomError): + with self.assertRaises(CustomError) as cm: self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing tuple item 2', + 'when serializing test.pickletester.REX reconstructor arguments', + 'when serializing test.pickletester.REX object']) def test_bad_newobj_args(self): obj = REX((copyreg.__newobj__, ())) @@ -1686,74 +1748,154 @@ def test_bad_newobj_args(self): with self.subTest(proto=proto): with self.assertRaises((IndexError, pickle.PicklingError)) as cm: self.dumps(obj, proto) + self.assertIn(str(cm.exception), { + 'tuple index out of range', + '__newobj__ expected at least 1 argument, got 0'}) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) obj = REX((copyreg.__newobj__, [REX])) for proto in protocols[2:]: with self.subTest(proto=proto): - with self.assertRaises((IndexError, pickle.PicklingError)): + with self.assertRaises(pickle.PicklingError) as cm: self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'second item of the tuple returned by __reduce__ ' + 'must be a tuple, not list') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) def test_bad_newobj_class(self): obj = REX((copyreg.__newobj__, (NoNew(),))) for proto in protocols[2:]: with self.subTest(proto=proto): - with self.assertRaises(pickle.PicklingError): + with self.assertRaises(pickle.PicklingError) as cm: self.dumps(obj, proto) + self.assertIn(str(cm.exception), { + 'first argument to __newobj__() has no __new__', + f'first argument to __newobj__() must be a class, not {__name__}.NoNew'}) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) def test_wrong_newobj_class(self): obj = REX((copyreg.__newobj__, (str,))) for proto in protocols[2:]: with self.subTest(proto=proto): - with self.assertRaises(pickle.PicklingError): + with self.assertRaises(pickle.PicklingError) as cm: self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f'first argument to __newobj__() must be {REX!r}, not {str!r}') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) def test_unpickleable_newobj_class(self): class LocalREX(REX): pass obj = LocalREX((copyreg.__newobj__, (LocalREX,))) for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises((pickle.PicklingError, AttributeError)): + with self.assertRaises(pickle.PicklingError) as cm: self.dumps(obj, proto) + if proto >= 2: + self.assertEqual(cm.exception.__notes__, [ + f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} class', + f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} object']) + else: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing tuple item 0', + f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} reconstructor arguments', + f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} object']) def test_unpickleable_newobj_args(self): obj = REX((copyreg.__newobj__, (REX, 1, 2, UNPICKLEABLE))) for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(CustomError): + with self.assertRaises(CustomError) as cm: self.dumps(obj, proto) + if proto >= 2: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing tuple item 2', + 'when serializing test.pickletester.REX __new__ arguments', + 'when serializing test.pickletester.REX object']) + else: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing tuple item 3', + 'when serializing test.pickletester.REX reconstructor arguments', + 'when serializing test.pickletester.REX object']) def test_bad_newobj_ex_args(self): obj = REX((copyreg.__newobj_ex__, ())) for proto in protocols[2:]: with self.subTest(proto=proto): - with self.assertRaises((ValueError, pickle.PicklingError)): + with self.assertRaises((ValueError, pickle.PicklingError)) as cm: self.dumps(obj, proto) + self.assertIn(str(cm.exception), { + 'not enough values to unpack (expected 3, got 0)', + '__newobj_ex__ expected 3 arguments, got 0'}) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) obj = REX((copyreg.__newobj_ex__, 42)) for proto in protocols[2:]: with self.subTest(proto=proto): - with self.assertRaises(pickle.PicklingError): + with self.assertRaises(pickle.PicklingError) as cm: self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'second item of the tuple returned by __reduce__ ' + 'must be a tuple, not int') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) obj = REX((copyreg.__newobj_ex__, (REX, 42, {}))) - is_py = self.pickler is pickle._Pickler - for proto in protocols[2:4] if is_py else protocols[2:]: - with self.subTest(proto=proto): - with self.assertRaises((TypeError, pickle.PicklingError)): - self.dumps(obj, proto) + if self.pickler is pickle._Pickler: + for proto in protocols[2:4]: + with self.subTest(proto=proto): + with self.assertRaises(TypeError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'Value after * must be an iterable, not int') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + else: + for proto in protocols[2:]: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'second argument to __newobj_ex__() must be a tuple, not int') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) obj = REX((copyreg.__newobj_ex__, (REX, (), []))) - for proto in protocols[2:4] if is_py else protocols[2:]: - with self.subTest(proto=proto): - with self.assertRaises((TypeError, pickle.PicklingError)): - self.dumps(obj, proto) + if self.pickler is pickle._Pickler: + for proto in protocols[2:4]: + with self.subTest(proto=proto): + with self.assertRaises(TypeError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'functools.partial() argument after ** must be a mapping, not list') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + else: + for proto in protocols[2:]: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'third argument to __newobj_ex__() must be a dict, not list') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) def test_bad_newobj_ex__class(self): obj = REX((copyreg.__newobj_ex__, (NoNew(), (), {}))) for proto in protocols[2:]: with self.subTest(proto=proto): - with self.assertRaises(pickle.PicklingError): + with self.assertRaises(pickle.PicklingError) as cm: self.dumps(obj, proto) + self.assertIn(str(cm.exception), { + 'first argument to __newobj_ex__() has no __new__', + f'first argument to __newobj_ex__() must be a class, not {__name__}.NoNew'}) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) def test_wrong_newobj_ex_class(self): if self.pickler is not pickle._Pickler: @@ -1761,37 +1903,99 @@ def test_wrong_newobj_ex_class(self): obj = REX((copyreg.__newobj_ex__, (str, (), {}))) for proto in protocols[2:]: with self.subTest(proto=proto): - with self.assertRaises(pickle.PicklingError): + with self.assertRaises(pickle.PicklingError) as cm: self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f'first argument to __newobj_ex__() must be {REX}, not {str}') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) def test_unpickleable_newobj_ex_class(self): class LocalREX(REX): pass obj = LocalREX((copyreg.__newobj_ex__, (LocalREX, (), {}))) for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises((pickle.PicklingError, AttributeError)): + with self.assertRaises(pickle.PicklingError) as cm: self.dumps(obj, proto) + if proto >= 4: + self.assertEqual(cm.exception.__notes__, [ + f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} class', + f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} object']) + elif proto >= 2: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing tuple item 0', + 'when serializing tuple item 1', + 'when serializing functools.partial state', + 'when serializing functools.partial object', + f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} reconstructor', + f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} object']) + else: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing tuple item 0', + f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} reconstructor arguments', + f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} object']) def test_unpickleable_newobj_ex_args(self): obj = REX((copyreg.__newobj_ex__, (REX, (1, 2, UNPICKLEABLE), {}))) for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(CustomError): + with self.assertRaises(CustomError) as cm: self.dumps(obj, proto) + if proto >= 4: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing tuple item 2', + 'when serializing test.pickletester.REX __new__ arguments', + 'when serializing test.pickletester.REX object']) + elif proto >= 2: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing tuple item 3', + 'when serializing tuple item 1', + 'when serializing functools.partial state', + 'when serializing functools.partial object', + 'when serializing test.pickletester.REX reconstructor', + 'when serializing test.pickletester.REX object']) + else: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing tuple item 2', + 'when serializing tuple item 1', + 'when serializing test.pickletester.REX reconstructor arguments', + 'when serializing test.pickletester.REX object']) def test_unpickleable_newobj_ex_kwargs(self): obj = REX((copyreg.__newobj_ex__, (REX, (), {'a': UNPICKLEABLE}))) for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(CustomError): + with self.assertRaises(CustomError) as cm: self.dumps(obj, proto) + if proto >= 4: + self.assertEqual(cm.exception.__notes__, [ + "when serializing dict item 'a'", + 'when serializing test.pickletester.REX __new__ arguments', + 'when serializing test.pickletester.REX object']) + elif proto >= 2: + self.assertEqual(cm.exception.__notes__, [ + "when serializing dict item 'a'", + 'when serializing tuple item 2', + 'when serializing functools.partial state', + 'when serializing functools.partial object', + 'when serializing test.pickletester.REX reconstructor', + 'when serializing test.pickletester.REX object']) + else: + self.assertEqual(cm.exception.__notes__, [ + "when serializing dict item 'a'", + 'when serializing tuple item 2', + 'when serializing test.pickletester.REX reconstructor arguments', + 'when serializing test.pickletester.REX object']) def test_unpickleable_state(self): obj = REX_state(UNPICKLEABLE) for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(CustomError): + with self.assertRaises(CustomError) as cm: self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX_state state', + 'when serializing test.pickletester.REX_state object']) def test_bad_state_setter(self): if self.pickler is pickle._Pickler: @@ -1799,22 +2003,33 @@ def test_bad_state_setter(self): obj = REX((print, (), 'state', None, None, 42)) for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(pickle.PicklingError): + with self.assertRaises(pickle.PicklingError) as cm: self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'sixth item of the tuple returned by __reduce__ ' + 'must be callable, not int') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) def test_unpickleable_state_setter(self): obj = REX((print, (), 'state', None, None, UnpickleableCallable())) for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(CustomError): + with self.assertRaises(CustomError) as cm: self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX state setter', + 'when serializing test.pickletester.REX object']) def test_unpickleable_state_with_state_setter(self): obj = REX((print, (), UNPICKLEABLE, None, None, print)) for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(CustomError): + with self.assertRaises(CustomError) as cm: self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX state', + 'when serializing test.pickletester.REX object']) def test_bad_object_list_items(self): # Issue4176: crash when 4th and 5th items of __reduce__() @@ -1822,23 +2037,37 @@ def test_bad_object_list_items(self): obj = REX((list, (), None, 42)) for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises((TypeError, pickle.PicklingError)): + with self.assertRaises((TypeError, pickle.PicklingError)) as cm: self.dumps(obj, proto) + self.assertIn(str(cm.exception), { + "'int' object is not iterable", + 'fourth item of the tuple returned by __reduce__ ' + 'must be an iterator, not int'}) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) if self.pickler is not pickle._Pickler: # Python implementation is less strict and also accepts iterables. obj = REX((list, (), None, [])) for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises((TypeError, pickle.PicklingError)): + with self.assertRaises(pickle.PicklingError): self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'fourth item of the tuple returned by __reduce__ ' + 'must be an iterator, not int') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) def test_unpickleable_object_list_items(self): obj = REX_six([1, 2, UNPICKLEABLE]) for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(CustomError): + with self.assertRaises(CustomError) as cm: self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX_six item 2', + 'when serializing test.pickletester.REX_six object']) def test_bad_object_dict_items(self): # Issue4176: crash when 4th and 5th items of __reduce__() @@ -1846,82 +2075,135 @@ def test_bad_object_dict_items(self): obj = REX((dict, (), None, None, 42)) for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises((TypeError, pickle.PicklingError)): + with self.assertRaises((TypeError, pickle.PicklingError)) as cm: self.dumps(obj, proto) + self.assertIn(str(cm.exception), { + "'int' object is not iterable", + 'fifth item of the tuple returned by __reduce__ ' + 'must be an iterator, not int'}) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) for proto in protocols: obj = REX((dict, (), None, None, iter([('a',)]))) with self.subTest(proto=proto): - with self.assertRaises((ValueError, TypeError)): + with self.assertRaises((ValueError, TypeError)) as cm: self.dumps(obj, proto) + self.assertIn(str(cm.exception), { + 'not enough values to unpack (expected 2, got 1)', + 'dict items iterator must return 2-tuples'}) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) if self.pickler is not pickle._Pickler: # Python implementation is less strict and also accepts iterables. obj = REX((dict, (), None, None, [])) for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises((TypeError, pickle.PicklingError)): + with self.assertRaises(pickle.PicklingError): self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'dict items iterator must return 2-tuples') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) def test_unpickleable_object_dict_items(self): obj = REX_seven({'a': UNPICKLEABLE}) for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(CustomError): + with self.assertRaises(CustomError) as cm: self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + "when serializing test.pickletester.REX_seven item 'a'", + 'when serializing test.pickletester.REX_seven object']) def test_unpickleable_list_items(self): obj = [1, [2, 3, UNPICKLEABLE]] for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(CustomError): + with self.assertRaises(CustomError) as cm: self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing list item 2', + 'when serializing list item 1']) for n in [0, 1, 1000, 1005]: obj = [*range(n), UNPICKLEABLE] for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(CustomError): + with self.assertRaises(CustomError) as cm: self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + f'when serializing list item {n}']) def test_unpickleable_tuple_items(self): obj = (1, (2, 3, UNPICKLEABLE)) for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(CustomError): + with self.assertRaises(CustomError) as cm: self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing tuple item 2', + 'when serializing tuple item 1']) obj = (*range(10), UNPICKLEABLE) for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(CustomError): + with self.assertRaises(CustomError) as cm: self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing tuple item 10']) def test_unpickleable_dict_items(self): obj = {'a': {'b': UNPICKLEABLE}} for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(CustomError): + with self.assertRaises(CustomError) as cm: self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + "when serializing dict item 'b'", + "when serializing dict item 'a'"]) for n in [0, 1, 1000, 1005]: obj = dict.fromkeys(range(n)) obj['a'] = UNPICKLEABLE for proto in protocols: with self.subTest(proto=proto, n=n): - with self.assertRaises(CustomError): + with self.assertRaises(CustomError) as cm: self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + "when serializing dict item 'a'"]) def test_unpickleable_set_items(self): obj = {UNPICKLEABLE} for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(CustomError): + with self.assertRaises(CustomError) as cm: self.dumps(obj, proto) + if proto >= 4: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing set element']) + else: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing list item 0', + 'when serializing tuple item 0', + 'when serializing set reconstructor arguments']) def test_unpickleable_frozenset_items(self): obj = frozenset({frozenset({UNPICKLEABLE})}) for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(CustomError): + with self.assertRaises(CustomError) as cm: self.dumps(obj, proto) + if proto >= 4: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing frozenset element', + 'when serializing frozenset element']) + else: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing list item 0', + 'when serializing tuple item 0', + 'when serializing frozenset reconstructor arguments', + 'when serializing list item 0', + 'when serializing tuple item 0', + 'when serializing frozenset reconstructor arguments']) def test_global_lookup_error(self): # Global name does not exist @@ -1929,26 +2211,42 @@ def test_global_lookup_error(self): obj.__module__ = __name__ for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(pickle.PicklingError): + with self.assertRaises(pickle.PicklingError) as cm: self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f"Can't pickle {obj!r}: it's not found as {__name__}.spam") + self.assertEqual(str(cm.exception.__context__), + f"module '{__name__}' has no attribute 'spam'") obj.__module__ = 'nonexisting' for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(pickle.PicklingError): + with self.assertRaises(pickle.PicklingError) as cm: self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f"Can't pickle {obj!r}: No module named 'nonexisting'") + self.assertEqual(str(cm.exception.__context__), + "No module named 'nonexisting'") obj.__module__ = '' for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises((ValueError, pickle.PicklingError)): + with self.assertRaises(pickle.PicklingError) as cm: self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f"Can't pickle {obj!r}: Empty module name") + self.assertEqual(str(cm.exception.__context__), + "Empty module name") obj.__module__ = None for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(pickle.PicklingError): + with self.assertRaises(pickle.PicklingError) as cm: self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f"Can't pickle {obj!r}: it's not found as __main__.spam") + self.assertEqual(str(cm.exception.__context__), + "module '__main__' has no attribute 'spam'") def test_nonencodable_global_name_error(self): for proto in protocols[:4]: @@ -1957,8 +2255,11 @@ def test_nonencodable_global_name_error(self): obj = REX(name) obj.__module__ = __name__ with support.swap_item(globals(), name, obj): - with self.assertRaises((UnicodeEncodeError, pickle.PicklingError)): + with self.assertRaises(pickle.PicklingError) as cm: self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f"can't pickle global identifier {name!r} using pickle protocol {proto}") + self.assertIsInstance(cm.exception.__context__, UnicodeEncodeError) def test_nonencodable_module_name_error(self): for proto in protocols[:4]: @@ -1968,8 +2269,11 @@ def test_nonencodable_module_name_error(self): obj.__module__ = name mod = types.SimpleNamespace(test=obj) with support.swap_item(sys.modules, name, mod): - with self.assertRaises((UnicodeEncodeError, pickle.PicklingError)): + with self.assertRaises(pickle.PicklingError) as cm: self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f"can't pickle module identifier {name!r} using pickle protocol {proto}") + self.assertIsInstance(cm.exception.__context__, UnicodeEncodeError) def test_nested_lookup_error(self): # Nested name does not exist @@ -1981,14 +2285,24 @@ class A: obj.__module__ = __name__ for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(pickle.PicklingError): + with self.assertRaises(pickle.PicklingError) as cm: self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f"Can't pickle {obj!r}: " + f"it's not found as {__name__}.TestGlobal.A.B.C") + self.assertEqual(str(cm.exception.__context__), + "type object 'A' has no attribute 'B'") obj.__module__ = None for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(pickle.PicklingError): + with self.assertRaises(pickle.PicklingError) as cm: self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f"Can't pickle {obj!r}: " + f"it's not found as __main__.TestGlobal.A.B.C") + self.assertEqual(str(cm.exception.__context__), + "module '__main__' has no attribute 'TestGlobal'") def test_wrong_object_lookup_error(self): # Name is bound to different object @@ -1999,14 +2313,23 @@ class TestGlobal: obj.__module__ = __name__ for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(pickle.PicklingError): + with self.assertRaises(pickle.PicklingError) as cm: self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f"Can't pickle {obj!r}: " + f"it's not the same object as {__name__}.TestGlobal") + self.assertIsNone(cm.exception.__context__) obj.__module__ = None for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(pickle.PicklingError): + with self.assertRaises(pickle.PicklingError) as cm: self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f"Can't pickle {obj!r}: " + f"it's not found as __main__.TestGlobal") + self.assertEqual(str(cm.exception.__context__), + "module '__main__' has no attribute 'TestGlobal'") def test_local_lookup_error(self): # Test that whichmodule() errors out cleanly when looking up @@ -2016,21 +2339,27 @@ def f(): # Since the function is local, lookup will fail for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises((AttributeError, pickle.PicklingError)): + with self.assertRaises(pickle.PicklingError) as cm: self.dumps(f, proto) + self.assertEqual(str(cm.exception), + f"Can't pickle local object {f!r}") # Same without a __module__ attribute (exercises a different path # in _pickle.c). del f.__module__ for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises((AttributeError, pickle.PicklingError)): + with self.assertRaises(pickle.PicklingError) as cm: self.dumps(f, proto) + self.assertEqual(str(cm.exception), + f"Can't pickle local object {f!r}") # Yet a different path. f.__name__ = f.__qualname__ for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises((AttributeError, pickle.PicklingError)): + with self.assertRaises(pickle.PicklingError) as cm: self.dumps(f, proto) + self.assertEqual(str(cm.exception), + f"Can't pickle local object {f!r}") def test_reduce_ex_None(self): c = REX_None() @@ -2744,7 +3073,7 @@ def test_proto(self): pickled = self.dumps(None, proto) if proto >= 2: proto_header = pickle.PROTO + bytes([proto]) - self.assertTrue(pickled.startswith(proto_header)) + self.assertStartsWith(pickled, proto_header) else: self.assertEqual(count_opcode(pickle.PROTO, pickled), 0) @@ -4640,8 +4969,11 @@ class MyClass: # NotImplemented self.assertIs(math_log, math.log) - with self.assertRaises(pickle.PicklingError): + with self.assertRaises(pickle.PicklingError) as cm: p.dump(g) + self.assertRegex(str(cm.exception), + r'(__reduce__|)' + r' must return (a )?string or tuple') with self.assertRaisesRegex( ValueError, 'The reducer just failed'): @@ -4680,7 +5012,7 @@ def test_default_dispatch_table(self): p = self.pickler_class(f, 0) with self.assertRaises(AttributeError): p.dispatch_table - self.assertFalse(hasattr(p, 'dispatch_table')) + self.assertNotHasAttr(p, 'dispatch_table') def test_class_dispatch_table(self): # A dispatch_table attribute can be specified class-wide diff --git a/Lib/test/profilee.py b/Lib/test/profilee.py new file mode 100644 index 00000000000..b6a090a2e34 --- /dev/null +++ b/Lib/test/profilee.py @@ -0,0 +1,115 @@ +""" +Input for test_profile.py and test_cprofile.py. + +IMPORTANT: This stuff is touchy. If you modify anything above the +test class you'll have to regenerate the stats by running the two +test files. + +*ALL* NUMBERS in the expected output are relevant. If you change +the formatting of pstats, please don't just regenerate the expected +output without checking very carefully that not a single number has +changed. +""" + +import sys + +# In order to have reproducible time, we simulate a timer in the global +# variable 'TICKS', which represents simulated time in milliseconds. +# (We can't use a helper function increment the timer since it would be +# included in the profile and would appear to consume all the time.) +TICKS = 42000 + +def timer(): + return TICKS + +def testfunc(): + # 1 call + # 1000 ticks total: 270 ticks local, 730 ticks in subfunctions + global TICKS + TICKS += 99 + helper() # 300 + helper() # 300 + TICKS += 171 + factorial(14) # 130 + +def factorial(n): + # 23 calls total + # 170 ticks total, 150 ticks local + # 3 primitive calls, 130, 20 and 20 ticks total + # including 116, 17, 17 ticks local + global TICKS + if n > 0: + TICKS += n + return mul(n, factorial(n-1)) + else: + TICKS += 11 + return 1 + +def mul(a, b): + # 20 calls + # 1 tick, local + global TICKS + TICKS += 1 + return a * b + +def helper(): + # 2 calls + # 300 ticks total: 20 ticks local, 260 ticks in subfunctions + global TICKS + TICKS += 1 + helper1() # 30 + TICKS += 2 + helper1() # 30 + TICKS += 6 + helper2() # 50 + TICKS += 3 + helper2() # 50 + TICKS += 2 + helper2() # 50 + TICKS += 5 + helper2_indirect() # 70 + TICKS += 1 + +def helper1(): + # 4 calls + # 30 ticks total: 29 ticks local, 1 tick in subfunctions + global TICKS + TICKS += 10 + hasattr(C(), "foo") # 1 + TICKS += 19 + lst = [] + lst.append(42) # 0 + sys.exception() # 0 + +def helper2_indirect(): + helper2() # 50 + factorial(3) # 20 + +def helper2(): + # 8 calls + # 50 ticks local: 39 ticks local, 11 ticks in subfunctions + global TICKS + TICKS += 11 + hasattr(C(), "bar") # 1 + TICKS += 13 + subhelper() # 10 + TICKS += 15 + +def subhelper(): + # 8 calls + # 10 ticks total: 8 ticks local, 2 ticks in subfunctions + global TICKS + TICKS += 2 + for i in range(2): # 0 + try: + C().foo # 1 x 2 + except AttributeError: + TICKS += 3 # 3 x 2 + +class C: + def __getattr__(self, name): + # 28 calls + # 1 tick, local + global TICKS + TICKS += 1 + raise AttributeError diff --git a/Lib/test/seq_tests.py b/Lib/test/seq_tests.py index 54eb5e65ac2..c7497d09f64 100644 --- a/Lib/test/seq_tests.py +++ b/Lib/test/seq_tests.py @@ -426,8 +426,7 @@ def test_pickle(self): self.assertEqual(lst2, lst) self.assertNotEqual(id(lst2), id(lst)) - @unittest.expectedFailure # TODO: RUSTPYTHON - @support.suppress_immortalization() + @unittest.expectedFailure # TODO: RUSTPYTHON def test_free_after_iterating(self): support.check_free_after_iterating(self, iter, self.type2test) support.check_free_after_iterating(self, reversed, self.type2test) diff --git a/Lib/test/ssl_servers.py b/Lib/test/ssl_servers.py index c45411f49d8..15b071e04dd 100644 --- a/Lib/test/ssl_servers.py +++ b/Lib/test/ssl_servers.py @@ -14,7 +14,7 @@ here = os.path.dirname(__file__) HOST = socket_helper.HOST -CERTFILE = os.path.join(here, 'certdata/keycert.pem') +CERTFILE = os.path.join(here, 'certdata', 'keycert.pem') # This one's based on HTTPServer, which is based on socketserver diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index cc5a48738fd..6b3f2c447e8 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -3,11 +3,12 @@ if __name__ != 'test.support': raise ImportError('support must be imported from the test package') -import _opcode +import annotationlib import contextlib import functools import inspect import logging +import _opcode import os import re import stat @@ -19,7 +20,6 @@ import unittest import warnings -import annotationlib __all__ = [ # globals @@ -30,7 +30,8 @@ "record_original_stdout", "get_original_stdout", "captured_stdout", "captured_stdin", "captured_stderr", "captured_output", # unittest - "is_resource_enabled", "requires", "requires_freebsd_version", + "is_resource_enabled", "get_resource_value", "requires", "requires_resource", + "requires_freebsd_version", "requires_gil_enabled", "requires_linux_version", "requires_mac_ver", "check_syntax_error", "requires_gzip", "requires_bz2", "requires_lzma", "requires_zstd", @@ -43,6 +44,7 @@ "check__all__", "skip_if_buggy_ucrt_strfptime", "check_disallow_instantiation", "check_sanitizer", "skip_if_sanitizer", "requires_limited_api", "requires_specialization", "thread_unsafe", + "skip_if_unlimited_stack_size", # sys "MS_WINDOWS", "is_jython", "is_android", "is_emscripten", "is_wasi", "is_apple_mobile", "check_impl_detail", "unix_shell", "setswitchinterval", @@ -184,7 +186,7 @@ def get_attribute(obj, name): return attribute verbose = 1 # Flag set to 0 by regrtest.py -use_resources = None # Flag set to [] by regrtest.py +use_resources = None # Flag set to {} by regrtest.py max_memuse = 0 # Disable bigmem tests (they will still be run with # small sizes, to make sure they work.) real_max_memuse = 0 @@ -299,6 +301,16 @@ def is_resource_enabled(resource): """ return use_resources is None or resource in use_resources +def get_resource_value(resource): + """Test whether a resource is enabled. + + Known resources are set by regrtest.py. If not running under regrtest.py, + all resources are assumed enabled unless use_resources has been set. + """ + if use_resources is None: + return None + return use_resources.get(resource) + def requires(resource, msg=None): """Raise ResourceDenied if the specified resource is not available.""" if not is_resource_enabled(resource): @@ -562,52 +574,6 @@ def requires_debug_ranges(reason='requires co_positions / debug_ranges'): return unittest.skipIf(skip, reason) -# XXX: RUSTPYTHON; this is not belong to 3.14 -def can_use_suppress_immortalization(suppress=True): - """Check if suppress_immortalization(suppress) can be used. - - Use this helper in code where SkipTest must be eagerly handled. - """ - if not suppress: - return True - try: - import _testinternalcapi - except ImportError: - return False - return True - - -@contextlib.contextmanager -def suppress_immortalization(suppress=True): - """Suppress immortalization of deferred objects. - - If _testinternalcapi is not available, the decorated test or class - is skipped. Use can_use_suppress_immortalization() outside test cases - to check if this decorator can be used. - """ - if not suppress: - yield # no-op - return - - from .import_helper import import_module - - _testinternalcapi = import_module("_testinternalcapi") - _testinternalcapi.suppress_immortalization(True) - try: - yield - finally: - _testinternalcapi.suppress_immortalization(False) - - -def skip_if_suppress_immortalization(): - try: - import _testinternalcapi - except ImportError: - return - return unittest.skipUnless(_testinternalcapi.get_immortalize_deferred(), - "requires immortalization of deferred objects") - - MS_WINDOWS = (sys.platform == 'win32') # Is not actually used in tests, but is kept for compatibility. @@ -783,9 +749,7 @@ def check_syntax_error(testcase, statement, errtext='', *, lineno=None, offset=N def open_urlresource(url, *args, **kw): - import urllib.parse - import urllib.request - + import urllib.request, urllib.parse from .os_helper import unlink try: import gzip @@ -919,13 +883,6 @@ def disable_gc(): @contextlib.contextmanager def gc_threshold(*args): - # TODO: RUSTPYTHON; GC is not supported yet - try: - yield - finally: - pass - return - import gc old_threshold = gc.get_threshold() gc.set_threshold(*args) @@ -1410,7 +1367,6 @@ def refcount_test(test): def requires_limited_api(test): try: import _testcapi # noqa: F401 - import _testlimitedcapi # noqa: F401 except ImportError: return unittest.skip('needs _testcapi and _testlimitedcapi modules')(test) @@ -1666,8 +1622,8 @@ def __init__(self, link=None): if sys.platform == "win32": def _platform_specific(self): - import _winapi import glob + import _winapi if os.path.lexists(self.real) and not os.path.exists(self.real): # App symlink appears to not exist, but we want the @@ -1739,6 +1695,25 @@ def skip_if_pgo_task(test): return test if ok else unittest.skip(msg)(test) +def skip_if_unlimited_stack_size(test): + """Skip decorator for tests not run when an unlimited stack size is configured. + + Tests using support.infinite_recursion([...]) may otherwise run into + an infinite loop, running until the memory on the system is filled and + crashing due to OOM. + + See https://github.com/python/cpython/issues/143460. + """ + if is_emscripten or is_wasi or os.name == "nt": + return test + + import resource + curlim, maxlim = resource.getrlimit(resource.RLIMIT_STACK) + unlimited_stack_size_cond = curlim == maxlim and curlim in (-1, 0xFFFF_FFFF_FFFF_FFFF) + reason = "Not run due to unlimited stack size" + return unittest.skipIf(unlimited_stack_size_cond, reason)(test) + + def detect_api_mismatch(ref_api, other_api, *, ignore=()): """Returns the set of items in ref_api not in other_api, except for a defined list of items to be ignored in this check. @@ -2063,10 +2038,9 @@ def missing_compiler_executable(cmd_names=[]): missing. """ - import shutil - - from setuptools import errors from setuptools._distutils import ccompiler, sysconfig + from setuptools import errors + import shutil compiler = ccompiler.new_compiler() sysconfig.customize_compiler(compiler) @@ -2556,7 +2530,6 @@ def _findwheel(pkgname): @contextlib.contextmanager def setup_venv_with_pip_setuptools(venv_dir): import subprocess - from .os_helper import temp_cwd def run_command(cmd): @@ -2763,15 +2736,6 @@ def adjust_int_max_str_digits(max_digits): sys.set_int_max_str_digits(current) -# XXX: RUSTPYTHON; removed in 3.14 -def get_c_recursion_limit(): - try: - import _testcapi - return _testcapi.Py_C_RECURSION_LIMIT - except ImportError: - raise unittest.SkipTest('requires _testcapi') - - def exceeds_recursion_limit(): """For recursion tests, easily exceeds default recursion limit.""" return 150_000 @@ -2988,7 +2952,6 @@ def is_slot_wrapper(name, value): @contextlib.contextmanager def force_color(color: bool): import _colorize - from .os_helper import EnvironmentVarGuard with ( @@ -3267,3 +3230,10 @@ def linked_to_musl(): return _linked_to_musl _linked_to_musl = tuple(map(int, version.split('.'))) return _linked_to_musl + + +def control_characters_c0() -> list[str]: + """Returns a list of C0 control characters as strings. + C0 control characters defined as the byte range 0x00-0x1F, and 0x7F. + """ + return [chr(c) for c in range(0x00, 0x20)] + ["\x7F"] diff --git a/Lib/test/support/import_helper.py b/Lib/test/support/import_helper.py index 2d80b663dd5..4c7eac0b7eb 100644 --- a/Lib/test/support/import_helper.py +++ b/Lib/test/support/import_helper.py @@ -1,5 +1,5 @@ -import _imp import contextlib +import _imp import importlib import importlib.machinery import importlib.util @@ -10,7 +10,7 @@ import unittest import warnings -from .os_helper import temp_dir, unlink +from .os_helper import unlink, temp_dir @contextlib.contextmanager diff --git a/Lib/test/support/rustpython.py b/Lib/test/support/rustpython.py new file mode 100644 index 00000000000..8ed7bc24dcf --- /dev/null +++ b/Lib/test/support/rustpython.py @@ -0,0 +1,24 @@ +""" +RustPython specific helpers. +""" + +import doctest + + +# copied from https://github.com/RustPython/RustPython/pull/6919 +EXPECTED_FAILURE = doctest.register_optionflag("EXPECTED_FAILURE") + + +class DocTestChecker(doctest.OutputChecker): + """ + Custom output checker that lets us add: `+EXPECTED_FAILURE` for doctest tests. + + We want to be able to mark failing doctest test the same way we do with normal + unit test, without this class we would have to skip the doctest for the CI to pass. + """ + + def check_output(self, want, got, optionflags): + res = super().check_output(want, got, optionflags) + if optionflags & EXPECTED_FAILURE: + res = not res + return res diff --git a/Lib/test/test___all__.py b/Lib/test/test___all__.py index 7b5356ea02a..8ded9f99248 100644 --- a/Lib/test/test___all__.py +++ b/Lib/test/test___all__.py @@ -3,7 +3,6 @@ from test.support import warnings_helper import os import sys -import types if support.check_sanitizer(address=True, memory=True): @@ -38,6 +37,7 @@ def check_all(self, modname): (".* (module|package)", DeprecationWarning), (".* (module|package)", PendingDeprecationWarning), ("", ResourceWarning), + ("", SyntaxWarning), quiet=True): try: exec("import %s" % modname, names) @@ -53,6 +53,7 @@ def check_all(self, modname): with warnings_helper.check_warnings( ("", DeprecationWarning), ("", ResourceWarning), + ("", SyntaxWarning), quiet=True): try: exec("from %s import *" % modname, names) @@ -71,6 +72,8 @@ def check_all(self, modname): all_set = set(all_list) self.assertCountEqual(all_set, all_list, "in module {}".format(modname)) self.assertEqual(keys, all_set, "in module {}".format(modname)) + # Verify __dir__ is non-empty and doesn't produce an error + self.assertTrue(dir(sys.modules[modname])) def walk_modules(self, basedir, modpath): for fn in sorted(os.listdir(basedir)): @@ -94,8 +97,6 @@ def walk_modules(self, basedir, modpath): continue yield path, modpath + modname - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_all(self): # List of denied modules and packages denylist = set([ @@ -106,7 +107,7 @@ def test_all(self): # In case _socket fails to build, make this test fail more gracefully # than an AttributeError somewhere deep in concurrent.futures, email # or unittest. - import _socket + import _socket # noqa: F401 ignored = [] failed_imports = [] diff --git a/Lib/test/test__locale.py b/Lib/test/test__locale.py new file mode 100644 index 00000000000..11b2c9545a1 --- /dev/null +++ b/Lib/test/test__locale.py @@ -0,0 +1,300 @@ +from _locale import (setlocale, LC_ALL, LC_CTYPE, LC_NUMERIC, LC_TIME, localeconv, Error) +try: + from _locale import (RADIXCHAR, THOUSEP, nl_langinfo) +except ImportError: + nl_langinfo = None + +import locale +import sys +import unittest +from platform import uname + +from test import support + +if uname().system == "Darwin": + maj, min, mic = [int(part) for part in uname().release.split(".")] + if (maj, min, mic) < (8, 0, 0): + raise unittest.SkipTest("locale support broken for OS X < 10.4") + +candidate_locales = ['es_UY', 'fr_FR', 'fi_FI', 'es_CO', 'pt_PT', 'it_IT', + 'et_EE', 'es_PY', 'no_NO', 'nl_NL', 'lv_LV', 'el_GR', 'be_BY', 'fr_BE', + 'ro_RO', 'ru_UA', 'ru_RU', 'es_VE', 'ca_ES', 'se_NO', 'es_EC', 'id_ID', + 'ka_GE', 'es_CL', 'wa_BE', 'hu_HU', 'lt_LT', 'sl_SI', 'hr_HR', 'es_AR', + 'es_ES', 'oc_FR', 'gl_ES', 'bg_BG', 'is_IS', 'mk_MK', 'de_AT', 'pt_BR', + 'da_DK', 'nn_NO', 'cs_CZ', 'de_LU', 'es_BO', 'sq_AL', 'sk_SK', 'fr_CH', + 'de_DE', 'sr_YU', 'br_FR', 'nl_BE', 'sv_FI', 'pl_PL', 'fr_CA', 'fo_FO', + 'bs_BA', 'fr_LU', 'kl_GL', 'fa_IR', 'de_BE', 'sv_SE', 'it_CH', 'uk_UA', + 'eu_ES', 'vi_VN', 'af_ZA', 'nb_NO', 'en_DK', 'tg_TJ', 'ps_AF', 'en_US', + 'fr_FR.ISO8859-1', 'fr_FR.UTF-8', 'fr_FR.ISO8859-15@euro', + 'ru_RU.KOI8-R', 'ko_KR.eucKR', + 'ja_JP.UTF-8', 'lzh_TW.UTF-8', 'my_MM.UTF-8', 'or_IN.UTF-8', 'shn_MM.UTF-8', + 'ar_AE.UTF-8', 'bn_IN.UTF-8', 'mr_IN.UTF-8', 'th_TH.TIS620', +] + +def setUpModule(): + global candidate_locales + # Issue #13441: Skip some locales (e.g. cs_CZ and hu_HU) on Solaris to + # workaround a mbstowcs() bug. For example, on Solaris, the hu_HU locale uses + # the locale encoding ISO-8859-2, the thousands separator is b'\xA0' and it is + # decoded as U+30000020 (an invalid character) by mbstowcs(). + if sys.platform == 'sunos5': + old_locale = locale.setlocale(locale.LC_ALL) + try: + locales = [] + for loc in candidate_locales: + try: + locale.setlocale(locale.LC_ALL, loc) + except Error: + continue + encoding = locale.getencoding() + try: + localeconv() + except Exception as err: + print("WARNING: Skip locale %s (encoding %s): [%s] %s" + % (loc, encoding, type(err), err)) + else: + locales.append(loc) + candidate_locales = locales + finally: + locale.setlocale(locale.LC_ALL, old_locale) + + # Workaround for MSVC6(debug) crash bug + if "MSC v.1200" in sys.version: + def accept(loc): + a = loc.split(".") + return not(len(a) == 2 and len(a[-1]) >= 9) + candidate_locales = [loc for loc in candidate_locales if accept(loc)] + +# List known locale values to test against when available. +# Dict formatted as `` : (, )``. If a +# value is not known, use '' . +known_numerics = { + 'en_US': ('.', ','), + 'de_DE' : (',', '.'), + # The French thousands separator may be a breaking or non-breaking space + # depending on the platform, so do not test it + 'fr_FR' : (',', ''), + 'ps_AF': ('\u066b', '\u066c'), +} + +known_alt_digits = { + 'C': (0, {}), + 'en_US': (0, {}), + 'fa_IR': (100, {0: '\u06f0\u06f0', 10: '\u06f1\u06f0', 99: '\u06f9\u06f9'}), + 'ja_JP': (100, {1: '\u4e00', 10: '\u5341', 99: '\u4e5d\u5341\u4e5d'}), + 'lzh_TW': (32, {0: '\u3007', 10: '\u5341', 31: '\u5345\u4e00'}), + 'my_MM': (100, {0: '\u1040\u1040', 10: '\u1041\u1040', 99: '\u1049\u1049'}), + 'or_IN': (100, {0: '\u0b66', 10: '\u0b67\u0b66', 99: '\u0b6f\u0b6f'}), + 'shn_MM': (100, {0: '\u1090\u1090', 10: '\u1091\u1090', 99: '\u1099\u1099'}), + 'ar_AE': (100, {0: '\u0660', 10: '\u0661\u0660', 99: '\u0669\u0669'}), + 'bn_IN': (100, {0: '\u09e6', 10: '\u09e7\u09e6', 99: '\u09ef\u09ef'}), +} + +known_era = { + 'C': (0, ''), + 'en_US': (0, ''), + 'ja_JP': (11, '+:1:2019/05/01:2019/12/31:令和:%EC元年'), + 'zh_TW': (3, '+:1:1912/01/01:1912/12/31:民國:%EC元年'), + 'th_TW': (1, '+:1:-543/01/01:+*:พ.ศ.:%EC %Ey'), +} + +if sys.platform == 'win32': + # ps_AF doesn't work on Windows: see bpo-38324 (msg361830) + del known_numerics['ps_AF'] + +if sys.platform == 'sunos5': + # On Solaris, Japanese ERAs start with the year 1927, + # and thus there's less of them. + known_era['ja_JP'] = (5, '+:1:2019/05/01:2019/12/31:令和:%EC元年') + +class _LocaleTests(unittest.TestCase): + + def setUp(self): + self.oldlocale = setlocale(LC_ALL) + + def tearDown(self): + setlocale(LC_ALL, self.oldlocale) + + # Want to know what value was calculated, what it was compared against, + # what function was used for the calculation, what type of data was used, + # the locale that was supposedly set, and the actual locale that is set. + lc_numeric_err_msg = "%s != %s (%s for %s; set to %s, using %s)" + + def numeric_tester(self, calc_type, calc_value, data_type, used_locale): + """Compare calculation against known value, if available""" + try: + set_locale = setlocale(LC_NUMERIC) + except Error: + set_locale = "" + known_value = known_numerics.get(used_locale, + ('', ''))[data_type == 'thousands_sep'] + if known_value and calc_value: + self.assertEqual(calc_value, known_value, + self.lc_numeric_err_msg % ( + calc_value, known_value, + calc_type, data_type, set_locale, + used_locale)) + return True + + @unittest.skipUnless(nl_langinfo, "nl_langinfo is not available") + @unittest.skipIf(support.linked_to_musl(), "musl libc issue, bpo-46390") + def test_lc_numeric_nl_langinfo(self): + # Test nl_langinfo against known values + tested = False + oldloc = setlocale(LC_CTYPE) + for loc in candidate_locales: + try: + setlocale(LC_NUMERIC, loc) + except Error: + continue + for li, lc in ((RADIXCHAR, "decimal_point"), + (THOUSEP, "thousands_sep")): + if self.numeric_tester('nl_langinfo', nl_langinfo(li), lc, loc): + tested = True + self.assertEqual(setlocale(LC_CTYPE), oldloc) + if not tested: + self.skipTest('no suitable locales') + + @unittest.skipIf(support.linked_to_musl(), "musl libc issue, bpo-46390") + def test_lc_numeric_localeconv(self): + # Test localeconv against known values + tested = False + oldloc = setlocale(LC_CTYPE) + for loc in candidate_locales: + try: + setlocale(LC_NUMERIC, loc) + except Error: + continue + formatting = localeconv() + for lc in ("decimal_point", + "thousands_sep"): + if self.numeric_tester('localeconv', formatting[lc], lc, loc): + tested = True + self.assertEqual(setlocale(LC_CTYPE), oldloc) + if not tested: + self.skipTest('no suitable locales') + + @unittest.skipUnless(nl_langinfo, "nl_langinfo is not available") + def test_lc_numeric_basic(self): + # Test nl_langinfo against localeconv + tested = False + oldloc = setlocale(LC_CTYPE) + for loc in candidate_locales: + try: + setlocale(LC_NUMERIC, loc) + except Error: + continue + for li, lc in ((RADIXCHAR, "decimal_point"), + (THOUSEP, "thousands_sep")): + nl_radixchar = nl_langinfo(li) + li_radixchar = localeconv()[lc] + try: + set_locale = setlocale(LC_NUMERIC) + except Error: + set_locale = "" + self.assertEqual(nl_radixchar, li_radixchar, + "%s (nl_langinfo) != %s (localeconv) " + "(set to %s, using %s)" % ( + nl_radixchar, li_radixchar, + loc, set_locale)) + tested = True + self.assertEqual(setlocale(LC_CTYPE), oldloc) + if not tested: + self.skipTest('no suitable locales') + + @unittest.skipUnless(nl_langinfo, "nl_langinfo is not available") + @unittest.skipUnless(hasattr(locale, 'ALT_DIGITS'), "requires locale.ALT_DIGITS") + @unittest.skipIf(support.linked_to_musl(), "musl libc issue, bpo-46390") + def test_alt_digits_nl_langinfo(self): + # Test nl_langinfo(ALT_DIGITS) + tested = False + for loc in candidate_locales: + with self.subTest(locale=loc): + try: + setlocale(LC_TIME, loc) + except Error: + self.skipTest(f'no locale {loc!r}') + continue + + with self.subTest(locale=loc): + alt_digits = nl_langinfo(locale.ALT_DIGITS) + self.assertIsInstance(alt_digits, str) + alt_digits = alt_digits.split(';') if alt_digits else [] + if alt_digits: + self.assertGreaterEqual(len(alt_digits), 10, alt_digits) + loc1 = loc.split('.', 1)[0] + if loc1 in known_alt_digits: + count, samples = known_alt_digits[loc1] + if count and not alt_digits: + self.skipTest(f'ALT_DIGITS is not set for locale {loc!r} on this platform') + self.assertEqual(len(alt_digits), count, alt_digits) + for i in samples: + self.assertEqual(alt_digits[i], samples[i]) + tested = True + if not tested: + self.skipTest('no suitable locales') + + @unittest.skipUnless(nl_langinfo, "nl_langinfo is not available") + @unittest.skipUnless(hasattr(locale, 'ERA'), "requires locale.ERA") + @unittest.skipIf(support.linked_to_musl(), "musl libc issue, bpo-46390") + def test_era_nl_langinfo(self): + # Test nl_langinfo(ERA) + tested = False + for loc in candidate_locales: + with self.subTest(locale=loc): + try: + setlocale(LC_TIME, loc) + except Error: + self.skipTest(f'no locale {loc!r}') + continue + + with self.subTest(locale=loc): + era = nl_langinfo(locale.ERA) + self.assertIsInstance(era, str) + if era: + self.assertEqual(era.count(':'), (era.count(';') + 1) * 5, era) + + loc1 = loc.split('.', 1)[0] + if loc1 in known_era: + count, sample = known_era[loc1] + if count: + if not era: + self.skipTest(f'ERA is not set for locale {loc!r} on this platform') + self.assertGreaterEqual(era.count(';') + 1, count) + self.assertIn(sample, era) + else: + self.assertEqual(era, '') + tested = True + if not tested: + self.skipTest('no suitable locales') + + def test_float_parsing(self): + # Bug #1391872: Test whether float parsing is okay on European + # locales. + tested = False + oldloc = setlocale(LC_CTYPE) + for loc in candidate_locales: + try: + setlocale(LC_NUMERIC, loc) + except Error: + continue + + # Ignore buggy locale databases. (Mac OS 10.4 and some other BSDs) + if loc == 'eu_ES' and localeconv()['decimal_point'] == "' ": + continue + + self.assertEqual(int(eval('3.14') * 100), 314, + "using eval('3.14') failed for %s" % loc) + self.assertEqual(int(float('3.14') * 100), 314, + "using float('3.14') failed for %s" % loc) + if localeconv()['decimal_point'] != '.': + self.assertRaises(ValueError, float, + localeconv()['decimal_point'].join(['1', '23'])) + tested = True + self.assertEqual(setlocale(LC_CTYPE), oldloc) + if not tested: + self.skipTest('no suitable locales') + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test__opcode.py b/Lib/test/test__opcode.py index 43d475baa5d..c253bc2be02 100644 --- a/Lib/test/test__opcode.py +++ b/Lib/test/test__opcode.py @@ -38,7 +38,6 @@ def test_is_valid(self): opcodes = [dis.opmap[opname] for opname in names] self.check_bool_function_result(_opcode.is_valid, opcodes, True) - @unittest.expectedFailure # TODO: RUSTPYTHON; KeyError: 'BINARY_OP_ADD_INT' def test_opmaps(self): def check_roundtrip(name, map): return self.assertEqual(opcode.opname[map[name]], name) @@ -116,7 +115,6 @@ def test_stack_effect_jump(self): class SpecializationStatsTests(unittest.TestCase): - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 'load_attr' not found in [] def test_specialization_stats(self): stat_names = ["success", "failure", "hit", "deferred", "miss", "deopt"] specialized_opcodes = [ diff --git a/Lib/test/test__osx_support.py b/Lib/test/test__osx_support.py index 4a14cb35213..0813c4804c1 100644 --- a/Lib/test/test__osx_support.py +++ b/Lib/test/test__osx_support.py @@ -20,12 +20,13 @@ def setUp(self): self.prog_name = 'bogus_program_xxxx' self.temp_path_dir = os.path.abspath(os.getcwd()) self.env = self.enterContext(os_helper.EnvironmentVarGuard()) - for cv in ('CFLAGS', 'LDFLAGS', 'CPPFLAGS', - 'BASECFLAGS', 'BLDSHARED', 'LDSHARED', 'CC', - 'CXX', 'PY_CFLAGS', 'PY_LDFLAGS', 'PY_CPPFLAGS', - 'PY_CORE_CFLAGS', 'PY_CORE_LDFLAGS'): - if cv in self.env: - self.env.unset(cv) + + self.env.unset( + 'CFLAGS', 'LDFLAGS', 'CPPFLAGS', + 'BASECFLAGS', 'BLDSHARED', 'LDSHARED', 'CC', + 'CXX', 'PY_CFLAGS', 'PY_LDFLAGS', 'PY_CPPFLAGS', + 'PY_CORE_CFLAGS', 'PY_CORE_LDFLAGS' + ) def add_expected_saved_initial_values(self, config_vars, expected_vars): # Ensure that the initial values for all modified config vars @@ -65,8 +66,8 @@ def test__find_build_tool(self): 'cc not found - check xcode-select') def test__get_system_version(self): - self.assertTrue(platform.mac_ver()[0].startswith( - _osx_support._get_system_version())) + self.assertStartsWith(platform.mac_ver()[0], + _osx_support._get_system_version()) def test__remove_original_values(self): config_vars = { diff --git a/Lib/test/test_abc.py b/Lib/test/test_abc.py index 92bd955855b..e29fc5a2394 100644 --- a/Lib/test/test_abc.py +++ b/Lib/test/test_abc.py @@ -20,7 +20,7 @@ def test_abstractproperty_basics(self): def foo(self): pass self.assertTrue(foo.__isabstractmethod__) def bar(self): pass - self.assertFalse(hasattr(bar, "__isabstractmethod__")) + self.assertNotHasAttr(bar, "__isabstractmethod__") class C(metaclass=abc_ABCMeta): @abc.abstractproperty @@ -89,7 +89,7 @@ def test_abstractmethod_basics(self): def foo(self): pass self.assertTrue(foo.__isabstractmethod__) def bar(self): pass - self.assertFalse(hasattr(bar, "__isabstractmethod__")) + self.assertNotHasAttr(bar, "__isabstractmethod__") def test_abstractproperty_basics(self): @property @@ -168,8 +168,7 @@ def method_two(self): msg = r"class C without an implementation for abstract methods 'method_one', 'method_two'" self.assertRaisesRegex(TypeError, msg, C) - # TODO: RUSTPYTHON; AssertionError: False is not true - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: False is not true def test_abstractmethod_integration(self): for abstractthing in [abc.abstractmethod, abc.abstractproperty, abc.abstractclassmethod, @@ -278,21 +277,21 @@ class A(metaclass=abc_ABCMeta): class B(object): pass b = B() - self.assertFalse(issubclass(B, A)) - self.assertFalse(issubclass(B, (A,))) + self.assertNotIsSubclass(B, A) + self.assertNotIsSubclass(B, (A,)) self.assertNotIsInstance(b, A) self.assertNotIsInstance(b, (A,)) B1 = A.register(B) - self.assertTrue(issubclass(B, A)) - self.assertTrue(issubclass(B, (A,))) + self.assertIsSubclass(B, A) + self.assertIsSubclass(B, (A,)) self.assertIsInstance(b, A) self.assertIsInstance(b, (A,)) self.assertIs(B1, B) class C(B): pass c = C() - self.assertTrue(issubclass(C, A)) - self.assertTrue(issubclass(C, (A,))) + self.assertIsSubclass(C, A) + self.assertIsSubclass(C, (A,)) self.assertIsInstance(c, A) self.assertIsInstance(c, (A,)) @@ -303,16 +302,16 @@ class A(metaclass=abc_ABCMeta): class B(object): pass b = B() - self.assertTrue(issubclass(B, A)) - self.assertTrue(issubclass(B, (A,))) + self.assertIsSubclass(B, A) + self.assertIsSubclass(B, (A,)) self.assertIsInstance(b, A) self.assertIsInstance(b, (A,)) @A.register class C(B): pass c = C() - self.assertTrue(issubclass(C, A)) - self.assertTrue(issubclass(C, (A,))) + self.assertIsSubclass(C, A) + self.assertIsSubclass(C, (A,)) self.assertIsInstance(c, A) self.assertIsInstance(c, (A,)) self.assertIs(C, A.register(C)) @@ -323,14 +322,14 @@ class A(metaclass=abc_ABCMeta): class B: pass b = B() - self.assertFalse(isinstance(b, A)) - self.assertFalse(isinstance(b, (A,))) + self.assertNotIsInstance(b, A) + self.assertNotIsInstance(b, (A,)) token_old = abc_get_cache_token() A.register(B) token_new = abc_get_cache_token() self.assertGreater(token_new, token_old) - self.assertTrue(isinstance(b, A)) - self.assertTrue(isinstance(b, (A,))) + self.assertIsInstance(b, A) + self.assertIsInstance(b, (A,)) def test_registration_builtins(self): class A(metaclass=abc_ABCMeta): @@ -338,18 +337,18 @@ class A(metaclass=abc_ABCMeta): A.register(int) self.assertIsInstance(42, A) self.assertIsInstance(42, (A,)) - self.assertTrue(issubclass(int, A)) - self.assertTrue(issubclass(int, (A,))) + self.assertIsSubclass(int, A) + self.assertIsSubclass(int, (A,)) class B(A): pass B.register(str) class C(str): pass self.assertIsInstance("", A) self.assertIsInstance("", (A,)) - self.assertTrue(issubclass(str, A)) - self.assertTrue(issubclass(str, (A,))) - self.assertTrue(issubclass(C, A)) - self.assertTrue(issubclass(C, (A,))) + self.assertIsSubclass(str, A) + self.assertIsSubclass(str, (A,)) + self.assertIsSubclass(C, A) + self.assertIsSubclass(C, (A,)) def test_registration_edge_cases(self): class A(metaclass=abc_ABCMeta): @@ -377,39 +376,39 @@ class A(metaclass=abc_ABCMeta): def test_registration_transitiveness(self): class A(metaclass=abc_ABCMeta): pass - self.assertTrue(issubclass(A, A)) - self.assertTrue(issubclass(A, (A,))) + self.assertIsSubclass(A, A) + self.assertIsSubclass(A, (A,)) class B(metaclass=abc_ABCMeta): pass - self.assertFalse(issubclass(A, B)) - self.assertFalse(issubclass(A, (B,))) - self.assertFalse(issubclass(B, A)) - self.assertFalse(issubclass(B, (A,))) + self.assertNotIsSubclass(A, B) + self.assertNotIsSubclass(A, (B,)) + self.assertNotIsSubclass(B, A) + self.assertNotIsSubclass(B, (A,)) class C(metaclass=abc_ABCMeta): pass A.register(B) class B1(B): pass - self.assertTrue(issubclass(B1, A)) - self.assertTrue(issubclass(B1, (A,))) + self.assertIsSubclass(B1, A) + self.assertIsSubclass(B1, (A,)) class C1(C): pass B1.register(C1) - self.assertFalse(issubclass(C, B)) - self.assertFalse(issubclass(C, (B,))) - self.assertFalse(issubclass(C, B1)) - self.assertFalse(issubclass(C, (B1,))) - self.assertTrue(issubclass(C1, A)) - self.assertTrue(issubclass(C1, (A,))) - self.assertTrue(issubclass(C1, B)) - self.assertTrue(issubclass(C1, (B,))) - self.assertTrue(issubclass(C1, B1)) - self.assertTrue(issubclass(C1, (B1,))) + self.assertNotIsSubclass(C, B) + self.assertNotIsSubclass(C, (B,)) + self.assertNotIsSubclass(C, B1) + self.assertNotIsSubclass(C, (B1,)) + self.assertIsSubclass(C1, A) + self.assertIsSubclass(C1, (A,)) + self.assertIsSubclass(C1, B) + self.assertIsSubclass(C1, (B,)) + self.assertIsSubclass(C1, B1) + self.assertIsSubclass(C1, (B1,)) C1.register(int) class MyInt(int): pass - self.assertTrue(issubclass(MyInt, A)) - self.assertTrue(issubclass(MyInt, (A,))) + self.assertIsSubclass(MyInt, A) + self.assertIsSubclass(MyInt, (A,)) self.assertIsInstance(42, A) self.assertIsInstance(42, (A,)) @@ -469,16 +468,16 @@ def __subclasshook__(cls, C): if cls is A: return 'foo' in C.__dict__ return NotImplemented - self.assertFalse(issubclass(A, A)) - self.assertFalse(issubclass(A, (A,))) + self.assertNotIsSubclass(A, A) + self.assertNotIsSubclass(A, (A,)) class B: foo = 42 - self.assertTrue(issubclass(B, A)) - self.assertTrue(issubclass(B, (A,))) + self.assertIsSubclass(B, A) + self.assertIsSubclass(B, (A,)) class C: spam = 42 - self.assertFalse(issubclass(C, A)) - self.assertFalse(issubclass(C, (A,))) + self.assertNotIsSubclass(C, A) + self.assertNotIsSubclass(C, (A,)) def test_all_new_methods_are_called(self): class A(metaclass=abc_ABCMeta): @@ -495,7 +494,7 @@ class C(A, B): self.assertEqual(B.counter, 1) def test_ABC_has___slots__(self): - self.assertTrue(hasattr(abc.ABC, '__slots__')) + self.assertHasAttr(abc.ABC, '__slots__') def test_tricky_new_works(self): def with_metaclass(meta, *bases): @@ -517,13 +516,14 @@ def foo(self): del A.foo self.assertEqual(A.__abstractmethods__, {'foo'}) - self.assertFalse(hasattr(A, 'foo')) + self.assertNotHasAttr(A, 'foo') abc.update_abstractmethods(A) self.assertEqual(A.__abstractmethods__, set()) A() + def test_update_new_abstractmethods(self): class A(metaclass=abc_ABCMeta): @abc.abstractmethod @@ -589,7 +589,7 @@ def updated_foo(self): A.foo = updated_foo abc.update_abstractmethods(A) A() - self.assertFalse(hasattr(A, '__abstractmethods__')) + self.assertNotHasAttr(A, '__abstractmethods__') def test_update_del_implementation(self): class A(metaclass=abc_ABCMeta): @@ -685,10 +685,16 @@ class B(A, metaclass=abc_ABCMeta, name="test"): return TestLegacyAPI, TestABC, TestABCWithInitSubclass -TestLegacyAPI_Py, TestABC_Py, TestABCWithInitSubclass_Py = test_factory(abc.ABCMeta, - abc.get_cache_token) -TestLegacyAPI_C, TestABC_C, TestABCWithInitSubclass_C = test_factory(_py_abc.ABCMeta, - _py_abc.get_cache_token) +TestLegacyAPI_Py, TestABC_Py, TestABCWithInitSubclass_Py = test_factory(_py_abc.ABCMeta, + _py_abc.get_cache_token) +TestLegacyAPI_C, TestABC_C, TestABCWithInitSubclass_C = test_factory(abc.ABCMeta, + abc.get_cache_token) + +# gh-130095: The _py_abc tests are not thread-safe when run with +# `--parallel-threads` +TestLegacyAPI_Py.__unittest_thread_unsafe__ = True +TestABC_Py.__unittest_thread_unsafe__ = True +TestABCWithInitSubclass_Py.__unittest_thread_unsafe__ = True if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_abstract_numbers.py b/Lib/test/test_abstract_numbers.py index 72232b670cd..cf071d2c933 100644 --- a/Lib/test/test_abstract_numbers.py +++ b/Lib/test/test_abstract_numbers.py @@ -24,11 +24,11 @@ def not_implemented(*args, **kwargs): class TestNumbers(unittest.TestCase): def test_int(self): - self.assertTrue(issubclass(int, Integral)) - self.assertTrue(issubclass(int, Rational)) - self.assertTrue(issubclass(int, Real)) - self.assertTrue(issubclass(int, Complex)) - self.assertTrue(issubclass(int, Number)) + self.assertIsSubclass(int, Integral) + self.assertIsSubclass(int, Rational) + self.assertIsSubclass(int, Real) + self.assertIsSubclass(int, Complex) + self.assertIsSubclass(int, Number) self.assertEqual(7, int(7).real) self.assertEqual(0, int(7).imag) @@ -38,11 +38,11 @@ def test_int(self): self.assertEqual(1, int(7).denominator) def test_float(self): - self.assertFalse(issubclass(float, Integral)) - self.assertFalse(issubclass(float, Rational)) - self.assertTrue(issubclass(float, Real)) - self.assertTrue(issubclass(float, Complex)) - self.assertTrue(issubclass(float, Number)) + self.assertNotIsSubclass(float, Integral) + self.assertNotIsSubclass(float, Rational) + self.assertIsSubclass(float, Real) + self.assertIsSubclass(float, Complex) + self.assertIsSubclass(float, Number) self.assertEqual(7.3, float(7.3).real) self.assertEqual(0, float(7.3).imag) @@ -50,11 +50,11 @@ def test_float(self): self.assertEqual(-7.3, float(-7.3).conjugate()) def test_complex(self): - self.assertFalse(issubclass(complex, Integral)) - self.assertFalse(issubclass(complex, Rational)) - self.assertFalse(issubclass(complex, Real)) - self.assertTrue(issubclass(complex, Complex)) - self.assertTrue(issubclass(complex, Number)) + self.assertNotIsSubclass(complex, Integral) + self.assertNotIsSubclass(complex, Rational) + self.assertNotIsSubclass(complex, Real) + self.assertIsSubclass(complex, Complex) + self.assertIsSubclass(complex, Number) c1, c2 = complex(3, 2), complex(4,1) # XXX: This is not ideal, but see the comment in math_trunc(). diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py new file mode 100644 index 00000000000..e89d6c0b161 --- /dev/null +++ b/Lib/test/test_annotationlib.py @@ -0,0 +1,2252 @@ +"""Tests for the annotations module.""" + +import textwrap +import annotationlib +import builtins +import collections +import functools +import itertools +import pickle +from string.templatelib import Template, Interpolation +import types +import typing +import sys +import unittest +from annotationlib import ( + Format, + ForwardRef, + get_annotations, + annotations_to_string, + type_repr, +) +from typing import Unpack, get_type_hints, List, Union + +from test import support +from test.support import import_helper +from test.test_inspect import inspect_stock_annotations +from test.test_inspect import inspect_stringized_annotations +from test.test_inspect import inspect_stringized_annotations_2 +from test.test_inspect import inspect_stringized_annotations_pep695 + + +def times_three(fn): + @functools.wraps(fn) + def wrapper(a, b): + return fn(a * 3, b * 3) + + return wrapper + + +class MyClass: + def __repr__(self): + return "my repr" + + +class TestFormat(unittest.TestCase): + def test_enum(self): + self.assertEqual(Format.VALUE.value, 1) + self.assertEqual(Format.VALUE, 1) + + self.assertEqual(Format.VALUE_WITH_FAKE_GLOBALS.value, 2) + self.assertEqual(Format.VALUE_WITH_FAKE_GLOBALS, 2) + + self.assertEqual(Format.FORWARDREF.value, 3) + self.assertEqual(Format.FORWARDREF, 3) + + self.assertEqual(Format.STRING.value, 4) + self.assertEqual(Format.STRING, 4) + + +class TestForwardRefFormat(unittest.TestCase): + def test_closure(self): + def inner(arg: x): + pass + + anno = get_annotations(inner, format=Format.FORWARDREF) + fwdref = anno["arg"] + self.assertIsInstance(fwdref, ForwardRef) + self.assertEqual(fwdref.__forward_arg__, "x") + with self.assertRaises(NameError): + fwdref.evaluate() + + x = 1 + self.assertEqual(fwdref.evaluate(), x) + + anno = get_annotations(inner, format=Format.FORWARDREF) + self.assertEqual(anno["arg"], x) + + def test_multiple_closure(self): + def inner(arg: x[y]): + pass + + fwdref = get_annotations(inner, format=Format.FORWARDREF)["arg"] + self.assertIsInstance(fwdref, ForwardRef) + self.assertEqual(fwdref.__forward_arg__, "x[y]") + with self.assertRaises(NameError): + fwdref.evaluate() + + y = str + fwdref = get_annotations(inner, format=Format.FORWARDREF)["arg"] + self.assertIsInstance(fwdref, ForwardRef) + extra_name, extra_val = next(iter(fwdref.__extra_names__.items())) + self.assertEqual(fwdref.__forward_arg__.replace(extra_name, extra_val.__name__), "x[str]") + with self.assertRaises(NameError): + fwdref.evaluate() + + x = list + self.assertEqual(fwdref.evaluate(), x[y]) + + fwdref = get_annotations(inner, format=Format.FORWARDREF)["arg"] + self.assertEqual(fwdref, x[y]) + + def test_function(self): + def f(x: int, y: doesntexist): + pass + + anno = get_annotations(f, format=Format.FORWARDREF) + self.assertIs(anno["x"], int) + fwdref = anno["y"] + self.assertIsInstance(fwdref, ForwardRef) + self.assertEqual(fwdref.__forward_arg__, "doesntexist") + with self.assertRaises(NameError): + fwdref.evaluate() + self.assertEqual(fwdref.evaluate(globals={"doesntexist": 1}), 1) + + def test_nonexistent_attribute(self): + def f( + x: some.module, + y: some[module], + z: some(module), + alpha: some | obj, + beta: +some, + gamma: some < obj, + delta: some | {obj: module}, + epsilon: some | {obj}, + zeta: some | [obj, module], + eta: some | (), + ): + pass + + anno = get_annotations(f, format=Format.FORWARDREF) + x_anno = anno["x"] + self.assertIsInstance(x_anno, ForwardRef) + self.assertEqual(x_anno, support.EqualToForwardRef("some.module", owner=f)) + + y_anno = anno["y"] + self.assertIsInstance(y_anno, ForwardRef) + self.assertEqual(y_anno, support.EqualToForwardRef("some[module]", owner=f)) + + z_anno = anno["z"] + self.assertIsInstance(z_anno, ForwardRef) + self.assertEqual(z_anno, support.EqualToForwardRef("some(module)", owner=f)) + + alpha_anno = anno["alpha"] + self.assertIsInstance(alpha_anno, ForwardRef) + self.assertEqual(alpha_anno, support.EqualToForwardRef("some | obj", owner=f)) + + beta_anno = anno["beta"] + self.assertIsInstance(beta_anno, ForwardRef) + self.assertEqual(beta_anno, support.EqualToForwardRef("+some", owner=f)) + + gamma_anno = anno["gamma"] + self.assertIsInstance(gamma_anno, ForwardRef) + self.assertEqual(gamma_anno, support.EqualToForwardRef("some < obj", owner=f)) + + delta_anno = anno["delta"] + self.assertIsInstance(delta_anno, ForwardRef) + self.assertEqual(delta_anno, support.EqualToForwardRef("some | {obj: module}", owner=f)) + + epsilon_anno = anno["epsilon"] + self.assertIsInstance(epsilon_anno, ForwardRef) + self.assertEqual(epsilon_anno, support.EqualToForwardRef("some | {obj}", owner=f)) + + zeta_anno = anno["zeta"] + self.assertIsInstance(zeta_anno, ForwardRef) + self.assertEqual(zeta_anno, support.EqualToForwardRef("some | [obj, module]", owner=f)) + + eta_anno = anno["eta"] + self.assertIsInstance(eta_anno, ForwardRef) + self.assertEqual(eta_anno, support.EqualToForwardRef("some | ()", owner=f)) + + def test_partially_nonexistent(self): + # These annotations start with a non-existent variable and then use + # global types with defined values. This partially evaluates by putting + # those globals into `fwdref.__extra_names__`. + def f( + x: obj | int, + y: container[int:obj, int], + z: dict_val | {str: int}, + alpha: set_val | {str, int}, + beta: obj | bool | int, + gamma: obj | call_func(int, kwd=bool), + ): + pass + + def func(*args, **kwargs): + return Union[*args, *(kwargs.values())] + + anno = get_annotations(f, format=Format.FORWARDREF) + globals_ = { + "obj": str, "container": list, "dict_val": {1: 2}, "set_val": {1, 2}, + "call_func": func + } + + x_anno = anno["x"] + self.assertIsInstance(x_anno, ForwardRef) + self.assertEqual(x_anno.evaluate(globals=globals_), str | int) + + y_anno = anno["y"] + self.assertIsInstance(y_anno, ForwardRef) + self.assertEqual(y_anno.evaluate(globals=globals_), list[int:str, int]) + + z_anno = anno["z"] + self.assertIsInstance(z_anno, ForwardRef) + self.assertEqual(z_anno.evaluate(globals=globals_), {1: 2} | {str: int}) + + alpha_anno = anno["alpha"] + self.assertIsInstance(alpha_anno, ForwardRef) + self.assertEqual(alpha_anno.evaluate(globals=globals_), {1, 2} | {str, int}) + + beta_anno = anno["beta"] + self.assertIsInstance(beta_anno, ForwardRef) + self.assertEqual(beta_anno.evaluate(globals=globals_), str | bool | int) + + gamma_anno = anno["gamma"] + self.assertIsInstance(gamma_anno, ForwardRef) + self.assertEqual(gamma_anno.evaluate(globals=globals_), str | func(int, kwd=bool)) + + def test_partially_nonexistent_union(self): + # Test unions with '|' syntax equal unions with typing.Union[] with some forwardrefs + class UnionForwardrefs: + pipe: str | undefined + union: Union[str, undefined] + + annos = get_annotations(UnionForwardrefs, format=Format.FORWARDREF) + + pipe = annos["pipe"] + self.assertIsInstance(pipe, ForwardRef) + self.assertEqual( + pipe.evaluate(globals={"undefined": int}), + str | int, + ) + union = annos["union"] + self.assertIsInstance(union, Union) + arg1, arg2 = typing.get_args(union) + self.assertIs(arg1, str) + self.assertEqual( + arg2, support.EqualToForwardRef("undefined", is_class=True, owner=UnionForwardrefs) + ) + + +class TestStringFormat(unittest.TestCase): + def test_closure(self): + x = 0 + + def inner(arg: x): + pass + + anno = get_annotations(inner, format=Format.STRING) + self.assertEqual(anno, {"arg": "x"}) + + def test_closure_undefined(self): + if False: + x = 0 + + def inner(arg: x): + pass + + anno = get_annotations(inner, format=Format.STRING) + self.assertEqual(anno, {"arg": "x"}) + + def test_function(self): + def f(x: int, y: doesntexist): + pass + + anno = get_annotations(f, format=Format.STRING) + self.assertEqual(anno, {"x": "int", "y": "doesntexist"}) + + def test_expressions(self): + def f( + add: a + b, + sub: a - b, + mul: a * b, + matmul: a @ b, + truediv: a / b, + mod: a % b, + lshift: a << b, + rshift: a >> b, + or_: a | b, + xor: a ^ b, + and_: a & b, + floordiv: a // b, + pow_: a**b, + lt: a < b, + le: a <= b, + eq: a == b, + ne: a != b, + gt: a > b, + ge: a >= b, + invert: ~a, + neg: -a, + pos: +a, + getitem: a[b], + getattr: a.b, + call: a(b, *c, d=e), # **kwargs are not supported + *args: *a, + ): + pass + + anno = get_annotations(f, format=Format.STRING) + self.assertEqual( + anno, + { + "add": "a + b", + "sub": "a - b", + "mul": "a * b", + "matmul": "a @ b", + "truediv": "a / b", + "mod": "a % b", + "lshift": "a << b", + "rshift": "a >> b", + "or_": "a | b", + "xor": "a ^ b", + "and_": "a & b", + "floordiv": "a // b", + "pow_": "a ** b", + "lt": "a < b", + "le": "a <= b", + "eq": "a == b", + "ne": "a != b", + "gt": "a > b", + "ge": "a >= b", + "invert": "~a", + "neg": "-a", + "pos": "+a", + "getitem": "a[b]", + "getattr": "a.b", + "call": "a(b, *c, d=e)", + "args": "*a", + }, + ) + + def test_reverse_ops(self): + def f( + radd: 1 + a, + rsub: 1 - a, + rmul: 1 * a, + rmatmul: 1 @ a, + rtruediv: 1 / a, + rmod: 1 % a, + rlshift: 1 << a, + rrshift: 1 >> a, + ror: 1 | a, + rxor: 1 ^ a, + rand: 1 & a, + rfloordiv: 1 // a, + rpow: 1**a, + ): + pass + + anno = get_annotations(f, format=Format.STRING) + self.assertEqual( + anno, + { + "radd": "1 + a", + "rsub": "1 - a", + "rmul": "1 * a", + "rmatmul": "1 @ a", + "rtruediv": "1 / a", + "rmod": "1 % a", + "rlshift": "1 << a", + "rrshift": "1 >> a", + "ror": "1 | a", + "rxor": "1 ^ a", + "rand": "1 & a", + "rfloordiv": "1 // a", + "rpow": "1 ** a", + }, + ) + + def test_template_str(self): + def f( + x: t"{a}", + y: list[t"{a}"], + z: t"{a:b} {c!r} {d!s:t}", + a: t"a{b}c{d}e{f}g", + b: t"{a:{1}}", + c: t"{a | b * c}", + gh138558: t"{ 0}", + ): pass + + annos = get_annotations(f, format=Format.STRING) + self.assertEqual(annos, { + "x": "t'{a}'", + "y": "list[t'{a}']", + "z": "t'{a:b} {c!r} {d!s:t}'", + "a": "t'a{b}c{d}e{f}g'", + # interpolations in the format spec are eagerly evaluated so we can't recover the source + "b": "t'{a:1}'", + "c": "t'{a | b * c}'", + "gh138558": "t'{ 0}'", + }) + + def g( + x: t"{a}", + ): ... + + annos = get_annotations(g, format=Format.FORWARDREF) + templ = annos["x"] + # Template and Interpolation don't have __eq__ so we have to compare manually + self.assertIsInstance(templ, Template) + self.assertEqual(templ.strings, ("", "")) + self.assertEqual(len(templ.interpolations), 1) + interp = templ.interpolations[0] + self.assertEqual(interp.value, support.EqualToForwardRef("a", owner=g)) + self.assertEqual(interp.expression, "a") + self.assertIsNone(interp.conversion) + self.assertEqual(interp.format_spec, "") + + def test_getitem(self): + def f(x: undef1[str, undef2]): + pass + anno = get_annotations(f, format=Format.STRING) + self.assertEqual(anno, {"x": "undef1[str, undef2]"}) + + anno = get_annotations(f, format=Format.FORWARDREF) + fwdref = anno["x"] + self.assertIsInstance(fwdref, ForwardRef) + self.assertEqual( + fwdref.evaluate(globals={"undef1": dict, "undef2": float}), dict[str, float] + ) + + def test_slice(self): + def f(x: a[b:c]): + pass + anno = get_annotations(f, format=Format.STRING) + self.assertEqual(anno, {"x": "a[b:c]"}) + + def f(x: a[b:c, d:e]): + pass + anno = get_annotations(f, format=Format.STRING) + self.assertEqual(anno, {"x": "a[b:c, d:e]"}) + + obj = slice(1, 1, 1) + def f(x: obj): + pass + anno = get_annotations(f, format=Format.STRING) + self.assertEqual(anno, {"x": "obj"}) + + def test_literals(self): + def f( + a: 1, + b: 1.0, + c: "hello", + d: b"hello", + e: True, + f: None, + g: ..., + h: 1j, + ): + pass + + anno = get_annotations(f, format=Format.STRING) + self.assertEqual( + anno, + { + "a": "1", + "b": "1.0", + "c": 'hello', + "d": "b'hello'", + "e": "True", + "f": "None", + "g": "...", + "h": "1j", + }, + ) + + def test_displays(self): + # Simple case first + def f(x: a[[int, str], float]): + pass + anno = get_annotations(f, format=Format.STRING) + self.assertEqual(anno, {"x": "a[[int, str], float]"}) + + def g( + w: a[[int, str], float], + x: a[{int}, 3], + y: a[{int: str}, 4], + z: a[(int, str), 5], + ): + pass + anno = get_annotations(g, format=Format.STRING) + self.assertEqual( + anno, + { + "w": "a[[int, str], float]", + "x": "a[{int}, 3]", + "y": "a[{int: str}, 4]", + "z": "a[(int, str), 5]", + }, + ) + + def test_nested_expressions(self): + def f( + nested: list[Annotated[set[int], "set of ints", 4j]], + set: {a + b}, # single element because order is not guaranteed + dict: {a + b: c + d, "key": e + g}, + list: [a, b, c], + tuple: (a, b, c), + slice: (a[b:c], a[b:c:d], a[:c], a[b:], a[:], a[::d], a[b::d]), + extended_slice: a[:, :, c:d], + unpack1: [*a], + unpack2: [*a, b, c], + ): + pass + + anno = get_annotations(f, format=Format.STRING) + self.assertEqual( + anno, + { + "nested": "list[Annotated[set[int], 'set of ints', 4j]]", + "set": "{a + b}", + "dict": "{a + b: c + d, 'key': e + g}", + "list": "[a, b, c]", + "tuple": "(a, b, c)", + "slice": "(a[b:c], a[b:c:d], a[:c], a[b:], a[:], a[::d], a[b::d])", + "extended_slice": "a[:, :, c:d]", + "unpack1": "[*a]", + "unpack2": "[*a, b, c]", + }, + ) + + def test_unsupported_operations(self): + format_msg = "Cannot stringify annotation containing string formatting" + + def f(fstring: f"{a}"): + pass + + with self.assertRaisesRegex(TypeError, format_msg): + get_annotations(f, format=Format.STRING) + + def f(fstring_format: f"{a:02d}"): + pass + + with self.assertRaisesRegex(TypeError, format_msg): + get_annotations(f, format=Format.STRING) + + def test_shenanigans(self): + # In cases like this we can't reconstruct the source; test that we do something + # halfway reasonable. + def f(x: x | (1).__class__, y: (1).__class__): + pass + + self.assertEqual( + get_annotations(f, format=Format.STRING), + {"x": "x | ", "y": ""}, + ) + + +class TestGetAnnotations(unittest.TestCase): + def test_builtin_type(self): + self.assertEqual(get_annotations(int), {}) + self.assertEqual(get_annotations(object), {}) + + def test_custom_metaclass(self): + class Meta(type): + pass + + class C(metaclass=Meta): + x: int + + self.assertEqual(get_annotations(C), {"x": int}) + + def test_missing_dunder_dict(self): + class NoDict(type): + @property + def __dict__(cls): + raise AttributeError + + b: str + + class C1(metaclass=NoDict): + a: int + + self.assertEqual(get_annotations(C1), {"a": int}) + self.assertEqual( + get_annotations(C1, format=Format.FORWARDREF), + {"a": int}, + ) + self.assertEqual( + get_annotations(C1, format=Format.STRING), + {"a": "int"}, + ) + self.assertEqual(get_annotations(NoDict), {"b": str}) + self.assertEqual( + get_annotations(NoDict, format=Format.FORWARDREF), + {"b": str}, + ) + self.assertEqual( + get_annotations(NoDict, format=Format.STRING), + {"b": "str"}, + ) + + def test_format(self): + def f1(a: int): + pass + + def f2(a: undefined): + pass + + self.assertEqual( + get_annotations(f1, format=Format.VALUE), + {"a": int}, + ) + self.assertEqual(get_annotations(f1, format=1), {"a": int}) + + fwd = support.EqualToForwardRef("undefined", owner=f2) + self.assertEqual( + get_annotations(f2, format=Format.FORWARDREF), + {"a": fwd}, + ) + self.assertEqual(get_annotations(f2, format=3), {"a": fwd}) + + self.assertEqual( + get_annotations(f1, format=Format.STRING), + {"a": "int"}, + ) + self.assertEqual(get_annotations(f1, format=4), {"a": "int"}) + + with self.assertRaises(ValueError): + get_annotations(f1, format=42) + + with self.assertRaisesRegex( + ValueError, + r"The VALUE_WITH_FAKE_GLOBALS format is for internal use only", + ): + get_annotations(f1, format=Format.VALUE_WITH_FAKE_GLOBALS) + + with self.assertRaisesRegex( + ValueError, + r"The VALUE_WITH_FAKE_GLOBALS format is for internal use only", + ): + get_annotations(f1, format=2) + + def test_custom_object_with_annotations(self): + class C: + def __init__(self): + self.__annotations__ = {"x": int, "y": str} + + self.assertEqual(get_annotations(C()), {"x": int, "y": str}) + + def test_custom_format_eval_str(self): + def foo(): + pass + + with self.assertRaises(ValueError): + get_annotations(foo, format=Format.FORWARDREF, eval_str=True) + get_annotations(foo, format=Format.STRING, eval_str=True) + + def test_stock_annotations(self): + def foo(a: int, b: str): + pass + + for format in (Format.VALUE, Format.FORWARDREF): + with self.subTest(format=format): + self.assertEqual( + get_annotations(foo, format=format), + {"a": int, "b": str}, + ) + self.assertEqual( + get_annotations(foo, format=Format.STRING), + {"a": "int", "b": "str"}, + ) + + foo.__annotations__ = {"a": "foo", "b": "str"} + for format in Format: + if format == Format.VALUE_WITH_FAKE_GLOBALS: + continue + with self.subTest(format=format): + self.assertEqual( + get_annotations(foo, format=format), + {"a": "foo", "b": "str"}, + ) + + self.assertEqual( + get_annotations(foo, eval_str=True, locals=locals()), + {"a": foo, "b": str}, + ) + self.assertEqual( + get_annotations(foo, eval_str=True, globals=locals()), + {"a": foo, "b": str}, + ) + + def test_stock_annotations_in_module(self): + isa = inspect_stock_annotations + + for kwargs in [ + {}, + {"eval_str": False}, + {"format": Format.VALUE}, + {"format": Format.FORWARDREF}, + {"format": Format.VALUE, "eval_str": False}, + {"format": Format.FORWARDREF, "eval_str": False}, + ]: + with self.subTest(**kwargs): + self.assertEqual(get_annotations(isa, **kwargs), {"a": int, "b": str}) + self.assertEqual( + get_annotations(isa.MyClass, **kwargs), + {"a": int, "b": str}, + ) + self.assertEqual( + get_annotations(isa.function, **kwargs), + {"a": int, "b": str, "return": isa.MyClass}, + ) + self.assertEqual( + get_annotations(isa.function2, **kwargs), + {"a": int, "b": "str", "c": isa.MyClass, "return": isa.MyClass}, + ) + self.assertEqual( + get_annotations(isa.function3, **kwargs), + {"a": "int", "b": "str", "c": "MyClass"}, + ) + self.assertEqual( + get_annotations(annotationlib, **kwargs), {} + ) # annotations module has no annotations + self.assertEqual(get_annotations(isa.UnannotatedClass, **kwargs), {}) + self.assertEqual( + get_annotations(isa.unannotated_function, **kwargs), + {}, + ) + + for kwargs in [ + {"eval_str": True}, + {"format": Format.VALUE, "eval_str": True}, + ]: + with self.subTest(**kwargs): + self.assertEqual(get_annotations(isa, **kwargs), {"a": int, "b": str}) + self.assertEqual( + get_annotations(isa.MyClass, **kwargs), + {"a": int, "b": str}, + ) + self.assertEqual( + get_annotations(isa.function, **kwargs), + {"a": int, "b": str, "return": isa.MyClass}, + ) + self.assertEqual( + get_annotations(isa.function2, **kwargs), + {"a": int, "b": str, "c": isa.MyClass, "return": isa.MyClass}, + ) + self.assertEqual( + get_annotations(isa.function3, **kwargs), + {"a": int, "b": str, "c": isa.MyClass}, + ) + self.assertEqual(get_annotations(annotationlib, **kwargs), {}) + self.assertEqual(get_annotations(isa.UnannotatedClass, **kwargs), {}) + self.assertEqual( + get_annotations(isa.unannotated_function, **kwargs), + {}, + ) + + self.assertEqual( + get_annotations(isa, format=Format.STRING), + {"a": "int", "b": "str"}, + ) + self.assertEqual( + get_annotations(isa.MyClass, format=Format.STRING), + {"a": "int", "b": "str"}, + ) + self.assertEqual( + get_annotations(isa.function, format=Format.STRING), + {"a": "int", "b": "str", "return": "MyClass"}, + ) + self.assertEqual( + get_annotations(isa.function2, format=Format.STRING), + {"a": "int", "b": "str", "c": "MyClass", "return": "MyClass"}, + ) + self.assertEqual( + get_annotations(isa.function3, format=Format.STRING), + {"a": "int", "b": "str", "c": "MyClass"}, + ) + self.assertEqual( + get_annotations(annotationlib, format=Format.STRING), + {}, + ) + self.assertEqual( + get_annotations(isa.UnannotatedClass, format=Format.STRING), + {}, + ) + self.assertEqual( + get_annotations(isa.unannotated_function, format=Format.STRING), + {}, + ) + + def test_stock_annotations_on_wrapper(self): + isa = inspect_stock_annotations + + wrapped = times_three(isa.function) + self.assertEqual(wrapped(1, "x"), isa.MyClass(3, "xxx")) + self.assertIsNot(wrapped.__globals__, isa.function.__globals__) + self.assertEqual( + get_annotations(wrapped), + {"a": int, "b": str, "return": isa.MyClass}, + ) + self.assertEqual( + get_annotations(wrapped, format=Format.FORWARDREF), + {"a": int, "b": str, "return": isa.MyClass}, + ) + self.assertEqual( + get_annotations(wrapped, format=Format.STRING), + {"a": "int", "b": "str", "return": "MyClass"}, + ) + self.assertEqual( + get_annotations(wrapped, eval_str=True), + {"a": int, "b": str, "return": isa.MyClass}, + ) + self.assertEqual( + get_annotations(wrapped, eval_str=False), + {"a": int, "b": str, "return": isa.MyClass}, + ) + + def test_stringized_annotations_in_module(self): + isa = inspect_stringized_annotations + for kwargs in [ + {}, + {"eval_str": False}, + {"format": Format.VALUE}, + {"format": Format.FORWARDREF}, + {"format": Format.STRING}, + {"format": Format.VALUE, "eval_str": False}, + {"format": Format.FORWARDREF, "eval_str": False}, + {"format": Format.STRING, "eval_str": False}, + ]: + with self.subTest(**kwargs): + self.assertEqual( + get_annotations(isa, **kwargs), + {"a": "int", "b": "str"}, + ) + self.assertEqual( + get_annotations(isa.MyClass, **kwargs), + {"a": "int", "b": "str"}, + ) + self.assertEqual( + get_annotations(isa.function, **kwargs), + {"a": "int", "b": "str", "return": "MyClass"}, + ) + self.assertEqual( + get_annotations(isa.function2, **kwargs), + {"a": "int", "b": "'str'", "c": "MyClass", "return": "MyClass"}, + ) + self.assertEqual( + get_annotations(isa.function3, **kwargs), + {"a": "'int'", "b": "'str'", "c": "'MyClass'"}, + ) + self.assertEqual(get_annotations(isa.UnannotatedClass, **kwargs), {}) + self.assertEqual( + get_annotations(isa.unannotated_function, **kwargs), + {}, + ) + + for kwargs in [ + {"eval_str": True}, + {"eval_str": True, "globals": isa.__dict__, "locals": {}}, + {"eval_str": True, "globals": {}, "locals": isa.__dict__}, + {"format": Format.VALUE, "eval_str": True}, + ]: + with self.subTest(**kwargs): + self.assertEqual(get_annotations(isa, **kwargs), {"a": int, "b": str}) + self.assertEqual( + get_annotations(isa.MyClass, **kwargs), + {"a": int, "b": str}, + ) + self.assertEqual( + get_annotations(isa.function, **kwargs), + {"a": int, "b": str, "return": isa.MyClass}, + ) + self.assertEqual( + get_annotations(isa.function2, **kwargs), + {"a": int, "b": "str", "c": isa.MyClass, "return": isa.MyClass}, + ) + self.assertEqual( + get_annotations(isa.function3, **kwargs), + {"a": "int", "b": "str", "c": "MyClass"}, + ) + self.assertEqual(get_annotations(isa.UnannotatedClass, **kwargs), {}) + self.assertEqual( + get_annotations(isa.unannotated_function, **kwargs), + {}, + ) + + def test_stringized_annotations_in_empty_module(self): + isa2 = inspect_stringized_annotations_2 + self.assertEqual(get_annotations(isa2), {}) + self.assertEqual(get_annotations(isa2, eval_str=True), {}) + self.assertEqual(get_annotations(isa2, eval_str=False), {}) + + def test_stringized_annotations_with_star_unpack(self): + def f(*args: "*tuple[int, ...]"): ... + self.assertEqual(get_annotations(f, eval_str=True), + {'args': (*tuple[int, ...],)[0]}) + def f(*args: " *tuple[int, ...]"): ... + self.assertEqual(get_annotations(f, eval_str=True), + {'args': (*tuple[int, ...],)[0]}) + + + def test_stringized_annotations_on_wrapper(self): + isa = inspect_stringized_annotations + wrapped = times_three(isa.function) + self.assertEqual(wrapped(1, "x"), isa.MyClass(3, "xxx")) + self.assertIsNot(wrapped.__globals__, isa.function.__globals__) + self.assertEqual( + get_annotations(wrapped), + {"a": "int", "b": "str", "return": "MyClass"}, + ) + self.assertEqual( + get_annotations(wrapped, eval_str=True), + {"a": int, "b": str, "return": isa.MyClass}, + ) + self.assertEqual( + get_annotations(wrapped, eval_str=False), + {"a": "int", "b": "str", "return": "MyClass"}, + ) + + def test_stringized_annotations_on_partial_wrapper(self): + isa = inspect_stringized_annotations + + def times_three_str(fn: typing.Callable[[str], isa.MyClass]): + @functools.wraps(fn) + def wrapper(b: "str") -> "MyClass": + return fn(b * 3) + + return wrapper + + wrapped = times_three_str(functools.partial(isa.function, 1)) + self.assertEqual(wrapped("x"), isa.MyClass(1, "xxx")) + self.assertIsNot(wrapped.__globals__, isa.function.__globals__) + self.assertEqual( + get_annotations(wrapped, eval_str=True), + {"b": str, "return": isa.MyClass}, + ) + self.assertEqual( + get_annotations(wrapped, eval_str=False), + {"b": "str", "return": "MyClass"}, + ) + + # If functools is not loaded, names will be evaluated in the current + # module instead of being unwrapped to the original. + functools_mod = sys.modules["functools"] + del sys.modules["functools"] + + self.assertEqual( + get_annotations(wrapped, eval_str=True), + {"b": str, "return": MyClass}, + ) + self.assertEqual( + get_annotations(wrapped, eval_str=False), + {"b": "str", "return": "MyClass"}, + ) + + sys.modules["functools"] = functools_mod + + def test_stringized_annotations_on_class(self): + isa = inspect_stringized_annotations + # test that local namespace lookups work + self.assertEqual( + get_annotations(isa.MyClassWithLocalAnnotations), + {"x": "mytype"}, + ) + self.assertEqual( + get_annotations(isa.MyClassWithLocalAnnotations, eval_str=True), + {"x": int}, + ) + + def test_stringized_annotations_on_custom_object(self): + class HasAnnotations: + @property + def __annotations__(self): + return {"x": "int"} + + ha = HasAnnotations() + self.assertEqual(get_annotations(ha), {"x": "int"}) + self.assertEqual(get_annotations(ha, eval_str=True), {"x": int}) + + def test_stringized_annotation_permutations(self): + def define_class(name, has_future, has_annos, base_text, extra_names=None): + lines = [] + if has_future: + lines.append("from __future__ import annotations") + lines.append(f"class {name}({base_text}):") + if has_annos: + lines.append(f" {name}_attr: int") + else: + lines.append(" pass") + code = "\n".join(lines) + ns = support.run_code(code, extra_names=extra_names) + return ns[name] + + def check_annotations(cls, has_future, has_annos): + if has_annos: + if has_future: + anno = "int" + else: + anno = int + self.assertEqual(get_annotations(cls), {f"{cls.__name__}_attr": anno}) + else: + self.assertEqual(get_annotations(cls), {}) + + for meta_future, base_future, child_future, meta_has_annos, base_has_annos, child_has_annos in itertools.product( + (False, True), + (False, True), + (False, True), + (False, True), + (False, True), + (False, True), + ): + with self.subTest( + meta_future=meta_future, + base_future=base_future, + child_future=child_future, + meta_has_annos=meta_has_annos, + base_has_annos=base_has_annos, + child_has_annos=child_has_annos, + ): + meta = define_class( + "Meta", + has_future=meta_future, + has_annos=meta_has_annos, + base_text="type", + ) + base = define_class( + "Base", + has_future=base_future, + has_annos=base_has_annos, + base_text="metaclass=Meta", + extra_names={"Meta": meta}, + ) + child = define_class( + "Child", + has_future=child_future, + has_annos=child_has_annos, + base_text="Base", + extra_names={"Base": base}, + ) + check_annotations(meta, meta_future, meta_has_annos) + check_annotations(base, base_future, base_has_annos) + check_annotations(child, child_future, child_has_annos) + + def test_modify_annotations(self): + def f(x: int): + pass + + self.assertEqual(get_annotations(f), {"x": int}) + self.assertEqual( + get_annotations(f, format=Format.FORWARDREF), + {"x": int}, + ) + + f.__annotations__["x"] = str + # The modification is reflected in VALUE (the default) + self.assertEqual(get_annotations(f), {"x": str}) + # ... and also in FORWARDREF, which tries __annotations__ if available + self.assertEqual( + get_annotations(f, format=Format.FORWARDREF), + {"x": str}, + ) + # ... but not in STRING which always uses __annotate__ + self.assertEqual( + get_annotations(f, format=Format.STRING), + {"x": "int"}, + ) + + def test_non_dict_annotations(self): + class WeirdAnnotations: + @property + def __annotations__(self): + return "not a dict" + + wa = WeirdAnnotations() + for format in Format: + if format == Format.VALUE_WITH_FAKE_GLOBALS: + continue + with ( + self.subTest(format=format), + self.assertRaisesRegex( + ValueError, r".*__annotations__ is neither a dict nor None" + ), + ): + get_annotations(wa, format=format) + + def test_annotations_on_custom_object(self): + class HasAnnotations: + @property + def __annotations__(self): + return {"x": int} + + ha = HasAnnotations() + self.assertEqual(get_annotations(ha, format=Format.VALUE), {"x": int}) + self.assertEqual(get_annotations(ha, format=Format.FORWARDREF), {"x": int}) + + self.assertEqual(get_annotations(ha, format=Format.STRING), {"x": "int"}) + + def test_raising_annotations_on_custom_object(self): + class HasRaisingAnnotations: + @property + def __annotations__(self): + return {"x": undefined} + + hra = HasRaisingAnnotations() + + with self.assertRaises(NameError): + get_annotations(hra, format=Format.VALUE) + + with self.assertRaises(NameError): + get_annotations(hra, format=Format.FORWARDREF) + + undefined = float + self.assertEqual(get_annotations(hra, format=Format.VALUE), {"x": float}) + + def test_forwardref_prefers_annotations(self): + class HasBoth: + @property + def __annotations__(self): + return {"x": int} + + @property + def __annotate__(self): + return lambda format: {"x": str} + + hb = HasBoth() + self.assertEqual(get_annotations(hb, format=Format.VALUE), {"x": int}) + self.assertEqual(get_annotations(hb, format=Format.FORWARDREF), {"x": int}) + self.assertEqual(get_annotations(hb, format=Format.STRING), {"x": str}) + + def test_only_annotate(self): + def f(x: int): + pass + + class OnlyAnnotate: + @property + def __annotate__(self): + return f.__annotate__ + + oa = OnlyAnnotate() + self.assertEqual(get_annotations(oa, format=Format.VALUE), {"x": int}) + self.assertEqual(get_annotations(oa, format=Format.FORWARDREF), {"x": int}) + self.assertEqual( + get_annotations(oa, format=Format.STRING), + {"x": "int"}, + ) + + def test_non_dict_annotate(self): + class WeirdAnnotate: + def __annotate__(self, *args, **kwargs): + return "not a dict" + + wa = WeirdAnnotate() + for format in Format: + if format == Format.VALUE_WITH_FAKE_GLOBALS: + continue + with ( + self.subTest(format=format), + self.assertRaisesRegex( + ValueError, r".*__annotate__ returned a non-dict" + ), + ): + get_annotations(wa, format=format) + + def test_no_annotations(self): + class CustomClass: + pass + + class MyCallable: + def __call__(self): + pass + + for format in Format: + if format == Format.VALUE_WITH_FAKE_GLOBALS: + continue + for obj in (None, 1, object(), CustomClass()): + with self.subTest(format=format, obj=obj): + with self.assertRaises(TypeError): + get_annotations(obj, format=format) + + # Callables and types with no annotations return an empty dict + for obj in (int, len, MyCallable()): + with self.subTest(format=format, obj=obj): + self.assertEqual(get_annotations(obj, format=format), {}) + + def test_pep695_generic_class_with_future_annotations(self): + ann_module695 = inspect_stringized_annotations_pep695 + A_annotations = get_annotations(ann_module695.A, eval_str=True) + A_type_params = ann_module695.A.__type_params__ + self.assertIs(A_annotations["x"], A_type_params[0]) + self.assertEqual(A_annotations["y"].__args__[0], Unpack[A_type_params[1]]) + self.assertIs(A_annotations["z"].__args__[0], A_type_params[2]) + + def test_pep695_generic_class_with_future_annotations_and_local_shadowing(self): + B_annotations = get_annotations( + inspect_stringized_annotations_pep695.B, eval_str=True + ) + self.assertEqual(B_annotations, {"x": int, "y": str, "z": bytes}) + + def test_pep695_generic_class_with_future_annotations_name_clash_with_global_vars( + self, + ): + ann_module695 = inspect_stringized_annotations_pep695 + C_annotations = get_annotations(ann_module695.C, eval_str=True) + self.assertEqual( + set(C_annotations.values()), set(ann_module695.C.__type_params__) + ) + + def test_pep_695_generic_function_with_future_annotations(self): + ann_module695 = inspect_stringized_annotations_pep695 + generic_func_annotations = get_annotations( + ann_module695.generic_function, eval_str=True + ) + func_t_params = ann_module695.generic_function.__type_params__ + self.assertEqual( + generic_func_annotations.keys(), {"x", "y", "z", "zz", "return"} + ) + self.assertIs(generic_func_annotations["x"], func_t_params[0]) + self.assertEqual(generic_func_annotations["y"], Unpack[func_t_params[1]]) + self.assertIs(generic_func_annotations["z"].__origin__, func_t_params[2]) + self.assertIs(generic_func_annotations["zz"].__origin__, func_t_params[2]) + + def test_pep_695_generic_function_with_future_annotations_name_clash_with_global_vars( + self, + ): + self.assertEqual( + set( + get_annotations( + inspect_stringized_annotations_pep695.generic_function_2, + eval_str=True, + ).values() + ), + set( + inspect_stringized_annotations_pep695.generic_function_2.__type_params__ + ), + ) + + def test_pep_695_generic_method_with_future_annotations(self): + ann_module695 = inspect_stringized_annotations_pep695 + generic_method_annotations = get_annotations( + ann_module695.D.generic_method, eval_str=True + ) + params = { + param.__name__: param + for param in ann_module695.D.generic_method.__type_params__ + } + self.assertEqual( + generic_method_annotations, + {"x": params["Foo"], "y": params["Bar"], "return": None}, + ) + + def test_pep_695_generic_method_with_future_annotations_name_clash_with_global_vars( + self, + ): + self.assertEqual( + set( + get_annotations( + inspect_stringized_annotations_pep695.D.generic_method_2, + eval_str=True, + ).values() + ), + set( + inspect_stringized_annotations_pep695.D.generic_method_2.__type_params__ + ), + ) + + def test_pep_695_generic_method_with_future_annotations_name_clash_with_global_and_local_vars( + self, + ): + self.assertEqual( + get_annotations(inspect_stringized_annotations_pep695.E, eval_str=True), + {"x": str}, + ) + + def test_pep_695_generics_with_future_annotations_nested_in_function(self): + results = inspect_stringized_annotations_pep695.nested() + + self.assertEqual( + set(results.F_annotations.values()), set(results.F.__type_params__) + ) + self.assertEqual( + set(results.F_meth_annotations.values()), + set(results.F.generic_method.__type_params__), + ) + self.assertNotEqual( + set(results.F_meth_annotations.values()), set(results.F.__type_params__) + ) + self.assertEqual( + set(results.F_meth_annotations.values()).intersection( + results.F.__type_params__ + ), + set(), + ) + + self.assertEqual(results.G_annotations, {"x": str}) + + self.assertEqual( + set(results.generic_func_annotations.values()), + set(results.generic_func.__type_params__), + ) + + def test_partial_evaluation(self): + def f( + x: builtins.undef, + y: list[int], + z: 1 + int, + a: builtins.int, + b: [builtins.undef, builtins.int], + ): + pass + + self.assertEqual( + get_annotations(f, format=Format.FORWARDREF), + { + "x": support.EqualToForwardRef("builtins.undef", owner=f), + "y": list[int], + "z": support.EqualToForwardRef("1 + int", owner=f), + "a": int, + "b": [ + support.EqualToForwardRef("builtins.undef", owner=f), + # We can't resolve this because we have to evaluate the whole annotation + support.EqualToForwardRef("builtins.int", owner=f), + ], + }, + ) + + self.assertEqual( + get_annotations(f, format=Format.STRING), + { + "x": "builtins.undef", + "y": "list[int]", + "z": "1 + int", + "a": "builtins.int", + "b": "[builtins.undef, builtins.int]", + }, + ) + + def test_partial_evaluation_error(self): + def f(x: range[1]): + pass + with self.assertRaisesRegex( + TypeError, "type 'range' is not subscriptable" + ): + f.__annotations__ + + self.assertEqual( + get_annotations(f, format=Format.FORWARDREF), + { + "x": support.EqualToForwardRef("range[1]", owner=f), + }, + ) + + def test_partial_evaluation_cell(self): + obj = object() + + class RaisesAttributeError: + attriberr: obj.missing + + anno = get_annotations(RaisesAttributeError, format=Format.FORWARDREF) + self.assertEqual( + anno, + { + "attriberr": support.EqualToForwardRef( + "obj.missing", is_class=True, owner=RaisesAttributeError + ) + }, + ) + + def test_nonlocal_in_annotation_scope(self): + class Demo: + nonlocal sequence_b + x: sequence_b + y: sequence_b[int] + + fwdrefs = get_annotations(Demo, format=Format.FORWARDREF) + + self.assertIsInstance(fwdrefs["x"], ForwardRef) + self.assertIsInstance(fwdrefs["y"], ForwardRef) + + sequence_b = list + self.assertIs(fwdrefs["x"].evaluate(), list) + self.assertEqual(fwdrefs["y"].evaluate(), list[int]) + + def test_raises_error_from_value(self): + # test that if VALUE is the only supported format, but raises an error + # that error is propagated from get_annotations + class DemoException(Exception): ... + + def annotate(format, /): + if format == Format.VALUE: + raise DemoException() + else: + raise NotImplementedError(format) + + def f(): ... + + f.__annotate__ = annotate + + for fmt in [Format.VALUE, Format.FORWARDREF, Format.STRING]: + with self.assertRaises(DemoException): + get_annotations(f, format=fmt) + + +class TestCallEvaluateFunction(unittest.TestCase): + def test_evaluation(self): + def evaluate(format, exc=NotImplementedError): + if format > 2: + raise exc + return undefined + + with self.assertRaises(NameError): + annotationlib.call_evaluate_function(evaluate, Format.VALUE) + self.assertEqual( + annotationlib.call_evaluate_function(evaluate, Format.FORWARDREF), + support.EqualToForwardRef("undefined"), + ) + self.assertEqual( + annotationlib.call_evaluate_function(evaluate, Format.STRING), + "undefined", + ) + + def test_fake_global_evaluation(self): + # This will raise an AttributeError + def evaluate_union(format, exc=NotImplementedError): + if format == Format.VALUE_WITH_FAKE_GLOBALS: + # Return a ForwardRef + return builtins.undefined | list[int] + raise exc + + self.assertEqual( + annotationlib.call_evaluate_function(evaluate_union, Format.FORWARDREF), + support.EqualToForwardRef("builtins.undefined | list[int]"), + ) + + # This will raise an AttributeError + def evaluate_intermediate(format, exc=NotImplementedError): + if format == Format.VALUE_WITH_FAKE_GLOBALS: + intermediate = builtins.undefined + # Return a literal + return intermediate is None + raise exc + + self.assertIs( + annotationlib.call_evaluate_function(evaluate_intermediate, Format.FORWARDREF), + False, + ) + + +class TestCallAnnotateFunction(unittest.TestCase): + # Tests for user defined annotate functions. + + # Format and NotImplementedError are provided as arguments so they exist in + # the fake globals namespace. + # This avoids non-matching conditions passing by being converted to stringifiers. + # See: https://github.com/python/cpython/issues/138764 + + def test_user_annotate_value(self): + def annotate(format, /): + if format == Format.VALUE: + return {"x": str} + else: + raise NotImplementedError(format) + + annotations = annotationlib.call_annotate_function( + annotate, + Format.VALUE, + ) + + self.assertEqual(annotations, {"x": str}) + + def test_user_annotate_forwardref_supported(self): + # If Format.FORWARDREF is supported prefer it over Format.VALUE + def annotate(format, /, __Format=Format, __NotImplementedError=NotImplementedError): + if format == __Format.VALUE: + return {'x': str} + elif format == __Format.VALUE_WITH_FAKE_GLOBALS: + return {'x': int} + elif format == __Format.FORWARDREF: + return {'x': float} + else: + raise __NotImplementedError(format) + + annotations = annotationlib.call_annotate_function( + annotate, + Format.FORWARDREF + ) + + self.assertEqual(annotations, {"x": float}) + + def test_user_annotate_forwardref_fakeglobals(self): + # If Format.FORWARDREF is not supported, use Format.VALUE_WITH_FAKE_GLOBALS + # before falling back to Format.VALUE + def annotate(format, /, __Format=Format, __NotImplementedError=NotImplementedError): + if format == __Format.VALUE: + return {'x': str} + elif format == __Format.VALUE_WITH_FAKE_GLOBALS: + return {'x': int} + else: + raise __NotImplementedError(format) + + annotations = annotationlib.call_annotate_function( + annotate, + Format.FORWARDREF + ) + + self.assertEqual(annotations, {"x": int}) + + def test_user_annotate_forwardref_value_fallback(self): + # If Format.FORWARDREF and Format.VALUE_WITH_FAKE_GLOBALS are not supported + # use Format.VALUE + def annotate(format, /, __Format=Format, __NotImplementedError=NotImplementedError): + if format == __Format.VALUE: + return {"x": str} + else: + raise __NotImplementedError(format) + + annotations = annotationlib.call_annotate_function( + annotate, + Format.FORWARDREF, + ) + + self.assertEqual(annotations, {"x": str}) + + def test_user_annotate_string_supported(self): + # If Format.STRING is supported prefer it over Format.VALUE + def annotate(format, /, __Format=Format, __NotImplementedError=NotImplementedError): + if format == __Format.VALUE: + return {'x': str} + elif format == __Format.VALUE_WITH_FAKE_GLOBALS: + return {'x': int} + elif format == __Format.STRING: + return {'x': "float"} + else: + raise __NotImplementedError(format) + + annotations = annotationlib.call_annotate_function( + annotate, + Format.STRING, + ) + + self.assertEqual(annotations, {"x": "float"}) + + def test_user_annotate_string_fakeglobals(self): + # If Format.STRING is not supported but Format.VALUE_WITH_FAKE_GLOBALS is + # prefer that over Format.VALUE + def annotate(format, /, __Format=Format, __NotImplementedError=NotImplementedError): + if format == __Format.VALUE: + return {'x': str} + elif format == __Format.VALUE_WITH_FAKE_GLOBALS: + return {'x': int} + else: + raise __NotImplementedError(format) + + annotations = annotationlib.call_annotate_function( + annotate, + Format.STRING, + ) + + self.assertEqual(annotations, {"x": "int"}) + + def test_user_annotate_string_value_fallback(self): + # If Format.STRING and Format.VALUE_WITH_FAKE_GLOBALS are not + # supported fall back to Format.VALUE and convert to strings + def annotate(format, /, __Format=Format, __NotImplementedError=NotImplementedError): + if format == __Format.VALUE: + return {"x": str} + else: + raise __NotImplementedError(format) + + annotations = annotationlib.call_annotate_function( + annotate, + Format.STRING, + ) + + self.assertEqual(annotations, {"x": "str"}) + + def test_condition_not_stringified(self): + # Make sure the first condition isn't evaluated as True by being converted + # to a _Stringifier + def annotate(format, /): + if format == Format.FORWARDREF: + return {"x": str} + else: + raise NotImplementedError(format) + + with self.assertRaises(NotImplementedError): + annotationlib.call_annotate_function(annotate, Format.STRING) + + def test_unsupported_formats(self): + def annotate(format, /): + if format == Format.FORWARDREF: + return {"x": str} + else: + raise NotImplementedError(format) + + with self.assertRaises(ValueError): + annotationlib.call_annotate_function(annotate, Format.VALUE_WITH_FAKE_GLOBALS) + + with self.assertRaises(RuntimeError): + annotationlib.call_annotate_function(annotate, Format.VALUE) + + with self.assertRaises(ValueError): + # Some non-Format value + annotationlib.call_annotate_function(annotate, 7) + + def test_error_from_value_raised(self): + # Test that the error from format.VALUE is raised + # if all formats fail + + class DemoException(Exception): ... + + def annotate(format, /): + if format == Format.VALUE: + raise DemoException() + else: + raise NotImplementedError(format) + + for fmt in [Format.VALUE, Format.FORWARDREF, Format.STRING]: + with self.assertRaises(DemoException): + annotationlib.call_annotate_function(annotate, format=fmt) + + +class MetaclassTests(unittest.TestCase): + def test_annotated_meta(self): + class Meta(type): + a: int + + class X(metaclass=Meta): + pass + + class Y(metaclass=Meta): + b: float + + self.assertEqual(get_annotations(Meta), {"a": int}) + self.assertEqual(Meta.__annotate__(Format.VALUE), {"a": int}) + + self.assertEqual(get_annotations(X), {}) + self.assertIs(X.__annotate__, None) + + self.assertEqual(get_annotations(Y), {"b": float}) + self.assertEqual(Y.__annotate__(Format.VALUE), {"b": float}) + + def test_unannotated_meta(self): + class Meta(type): + pass + + class X(metaclass=Meta): + a: str + + class Y(X): + pass + + self.assertEqual(get_annotations(Meta), {}) + self.assertIs(Meta.__annotate__, None) + + self.assertEqual(get_annotations(Y), {}) + self.assertIs(Y.__annotate__, None) + + self.assertEqual(get_annotations(X), {"a": str}) + self.assertEqual(X.__annotate__(Format.VALUE), {"a": str}) + + def test_ordering(self): + # Based on a sample by David Ellis + # https://discuss.python.org/t/pep-749-implementing-pep-649/54974/38 + + def make_classes(): + class Meta(type): + a: int + expected_annotations = {"a": int} + + class A(type, metaclass=Meta): + b: float + expected_annotations = {"b": float} + + class B(metaclass=A): + c: str + expected_annotations = {"c": str} + + class C(B): + expected_annotations = {} + + class D(metaclass=Meta): + expected_annotations = {} + + return Meta, A, B, C, D + + classes = make_classes() + class_count = len(classes) + for order in itertools.permutations(range(class_count), class_count): + names = ", ".join(classes[i].__name__ for i in order) + with self.subTest(names=names): + classes = make_classes() # Regenerate classes + for i in order: + get_annotations(classes[i]) + for c in classes: + with self.subTest(c=c): + self.assertEqual(get_annotations(c), c.expected_annotations) + annotate_func = getattr(c, "__annotate__", None) + if c.expected_annotations: + self.assertEqual( + annotate_func(Format.VALUE), c.expected_annotations + ) + else: + self.assertIs(annotate_func, None) + + +class TestGetAnnotateFromClassNamespace(unittest.TestCase): + def test_with_metaclass(self): + class Meta(type): + def __new__(mcls, name, bases, ns): + annotate = annotationlib.get_annotate_from_class_namespace(ns) + expected = ns["expected_annotate"] + with self.subTest(name=name): + if expected: + self.assertIsNotNone(annotate) + else: + self.assertIsNone(annotate) + return super().__new__(mcls, name, bases, ns) + + class HasAnnotations(metaclass=Meta): + expected_annotate = True + a: int + + class NoAnnotations(metaclass=Meta): + expected_annotate = False + + class CustomAnnotate(metaclass=Meta): + expected_annotate = True + def __annotate__(format): + return {} + + code = """ + from __future__ import annotations + + class HasFutureAnnotations(metaclass=Meta): + expected_annotate = False + a: int + """ + exec(textwrap.dedent(code), {"Meta": Meta}) + + +class TestTypeRepr(unittest.TestCase): + def test_type_repr(self): + class Nested: + pass + + def nested(): + pass + + self.assertEqual(type_repr(int), "int") + self.assertEqual(type_repr(MyClass), f"{__name__}.MyClass") + self.assertEqual( + type_repr(Nested), f"{__name__}.TestTypeRepr.test_type_repr..Nested" + ) + self.assertEqual( + type_repr(nested), f"{__name__}.TestTypeRepr.test_type_repr..nested" + ) + self.assertEqual(type_repr(len), "len") + self.assertEqual(type_repr(type_repr), "annotationlib.type_repr") + self.assertEqual(type_repr(times_three), f"{__name__}.times_three") + self.assertEqual(type_repr(...), "...") + self.assertEqual(type_repr(None), "None") + self.assertEqual(type_repr(1), "1") + self.assertEqual(type_repr("1"), "'1'") + self.assertEqual(type_repr(Format.VALUE), repr(Format.VALUE)) + self.assertEqual(type_repr(MyClass()), "my repr") + # gh138558 tests + self.assertEqual(type_repr(t'''{ 0 + & 1 + | 2 + }'''), 't"""{ 0\n & 1\n | 2}"""') + self.assertEqual( + type_repr(Template("hi", Interpolation(42, "42"))), "t'hi{42}'" + ) + self.assertEqual( + type_repr(Template("hi", Interpolation(42))), + "Template('hi', Interpolation(42, '', None, ''))", + ) + self.assertEqual( + type_repr(Template("hi", Interpolation(42, " "))), + "Template('hi', Interpolation(42, ' ', None, ''))", + ) + # gh138558: perhaps in the future, we can improve this behavior: + self.assertEqual(type_repr(Template(Interpolation(42, "99"))), "t'{99}'") + + +class TestAnnotationsToString(unittest.TestCase): + def test_annotations_to_string(self): + self.assertEqual(annotations_to_string({}), {}) + self.assertEqual(annotations_to_string({"x": int}), {"x": "int"}) + self.assertEqual(annotations_to_string({"x": "int"}), {"x": "int"}) + self.assertEqual( + annotations_to_string({"x": int, "y": str}), {"x": "int", "y": "str"} + ) + + +class A: + pass + +TypeParamsAlias1 = int + +class TypeParamsSample[TypeParamsAlias1, TypeParamsAlias2]: + TypeParamsAlias2 = str + + +class TestForwardRefClass(unittest.TestCase): + def test_forwardref_instance_type_error(self): + fr = ForwardRef("int") + with self.assertRaises(TypeError): + isinstance(42, fr) + + def test_forwardref_subclass_type_error(self): + fr = ForwardRef("int") + with self.assertRaises(TypeError): + issubclass(int, fr) + + def test_forwardref_only_str_arg(self): + with self.assertRaises(TypeError): + ForwardRef(1) # only `str` type is allowed + + def test_forward_equality(self): + fr = ForwardRef("int") + self.assertEqual(fr, ForwardRef("int")) + self.assertNotEqual(List["int"], List[int]) + self.assertNotEqual(fr, ForwardRef("int", module=__name__)) + frm = ForwardRef("int", module=__name__) + self.assertEqual(frm, ForwardRef("int", module=__name__)) + self.assertNotEqual(frm, ForwardRef("int", module="__other_name__")) + + def test_forward_equality_get_type_hints(self): + c1 = ForwardRef("C") + c1_gth = ForwardRef("C") + c2 = ForwardRef("C") + c2_gth = ForwardRef("C") + + class C: + pass + + def foo(a: c1_gth, b: c2_gth): + pass + + self.assertEqual(get_type_hints(foo, globals(), locals()), {"a": C, "b": C}) + self.assertEqual(c1, c2) + self.assertEqual(c1, c1_gth) + self.assertEqual(c1_gth, c2_gth) + self.assertEqual(List[c1], List[c1_gth]) + self.assertNotEqual(List[c1], List[C]) + self.assertNotEqual(List[c1_gth], List[C]) + self.assertEqual(Union[c1, c1_gth], Union[c1]) + self.assertEqual(Union[c1, c1_gth, int], Union[c1, int]) + + def test_forward_equality_hash(self): + c1 = ForwardRef("int") + c1_gth = ForwardRef("int") + c2 = ForwardRef("int") + c2_gth = ForwardRef("int") + + def foo(a: c1_gth, b: c2_gth): + pass + + get_type_hints(foo, globals(), locals()) + + self.assertEqual(hash(c1), hash(c2)) + self.assertEqual(hash(c1_gth), hash(c2_gth)) + self.assertEqual(hash(c1), hash(c1_gth)) + + c3 = ForwardRef("int", module=__name__) + c4 = ForwardRef("int", module="__other_name__") + + self.assertNotEqual(hash(c3), hash(c1)) + self.assertNotEqual(hash(c3), hash(c1_gth)) + self.assertNotEqual(hash(c3), hash(c4)) + self.assertEqual(hash(c3), hash(ForwardRef("int", module=__name__))) + + def test_forward_equality_and_hash_with_cells(self): + """Regression test for GH-143831.""" + class A: + def one(_) -> C1: + """One cell.""" + + one_f = ForwardRef("C1", owner=one) + one_f_ga1 = get_annotations(one, format=Format.FORWARDREF)["return"] + one_f_ga2 = get_annotations(one, format=Format.FORWARDREF)["return"] + self.assertIsInstance(one_f_ga1.__cell__, types.CellType) + self.assertIs(one_f_ga1.__cell__, one_f_ga2.__cell__) + + def two(_) -> C1 | C2: + """Two cells.""" + + two_f_ga1 = get_annotations(two, format=Format.FORWARDREF)["return"] + two_f_ga2 = get_annotations(two, format=Format.FORWARDREF)["return"] + self.assertIsNot(two_f_ga1.__cell__, two_f_ga2.__cell__) + self.assertIsInstance(two_f_ga1.__cell__, dict) + self.assertIsInstance(two_f_ga2.__cell__, dict) + + type C1 = None + type C2 = None + + self.assertNotEqual(A.one_f, A.one_f_ga1) + self.assertNotEqual(hash(A.one_f), hash(A.one_f_ga1)) + + self.assertEqual(A.one_f_ga1, A.one_f_ga2) + self.assertEqual(hash(A.one_f_ga1), hash(A.one_f_ga2)) + + self.assertEqual(A.two_f_ga1, A.two_f_ga2) + self.assertEqual(hash(A.two_f_ga1), hash(A.two_f_ga2)) + + def test_forward_equality_namespace(self): + def namespace1(): + a = ForwardRef("A") + + def fun(x: a): + pass + + get_type_hints(fun, globals(), locals()) + return a + + def namespace2(): + a = ForwardRef("A") + + class A: + pass + + def fun(x: a): + pass + + get_type_hints(fun, globals(), locals()) + return a + + self.assertEqual(namespace1(), namespace1()) + self.assertEqual(namespace1(), namespace2()) + + def test_forward_repr(self): + self.assertEqual(repr(List["int"]), "typing.List[ForwardRef('int')]") + self.assertEqual( + repr(List[ForwardRef("int", module="mod")]), + "typing.List[ForwardRef('int', module='mod')]", + ) + self.assertEqual( + repr(List[ForwardRef("int", module="mod", is_class=True)]), + "typing.List[ForwardRef('int', module='mod', is_class=True)]", + ) + self.assertEqual( + repr(List[ForwardRef("int", owner="class")]), + "typing.List[ForwardRef('int', owner='class')]", + ) + + def test_forward_recursion_actually(self): + def namespace1(): + a = ForwardRef("A") + A = a + + def fun(x: a): + pass + + ret = get_type_hints(fun, globals(), locals()) + return a + + def namespace2(): + a = ForwardRef("A") + A = a + + def fun(x: a): + pass + + ret = get_type_hints(fun, globals(), locals()) + return a + + r1 = namespace1() + r2 = namespace2() + self.assertIsNot(r1, r2) + self.assertEqual(r1, r2) + + def test_syntax_error(self): + + with self.assertRaises(SyntaxError): + typing.Generic["/T"] + + def test_delayed_syntax_error(self): + + def foo(a: "Node[T"): + pass + + with self.assertRaises(SyntaxError): + get_type_hints(foo) + + def test_syntax_error_empty_string(self): + for form in [typing.List, typing.Set, typing.Type, typing.Deque]: + with self.subTest(form=form): + with self.assertRaises(SyntaxError): + form[""] + + def test_or(self): + X = ForwardRef("X") + # __or__/__ror__ itself + self.assertEqual(X | "x", Union[X, "x"]) + self.assertEqual("x" | X, Union["x", X]) + + def test_multiple_ways_to_create(self): + X1 = Union["X"] + self.assertIsInstance(X1, ForwardRef) + X2 = ForwardRef("X") + self.assertIsInstance(X2, ForwardRef) + self.assertEqual(X1, X2) + + def test_special_attrs(self): + # Forward refs provide a different introspection API. __name__ and + # __qualname__ make little sense for forward refs as they can store + # complex typing expressions. + fr = ForwardRef("set[Any]") + self.assertNotHasAttr(fr, "__name__") + self.assertNotHasAttr(fr, "__qualname__") + self.assertEqual(fr.__module__, "annotationlib") + # Forward refs are currently unpicklable once they contain a code object. + fr.__forward_code__ # fill the cache + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.assertRaises(TypeError): + pickle.dumps(fr, proto) + + def test_evaluate_string_format(self): + fr = ForwardRef("set[Any]") + self.assertEqual(fr.evaluate(format=Format.STRING), "set[Any]") + + def test_evaluate_forwardref_format(self): + fr = ForwardRef("undef") + evaluated = fr.evaluate(format=Format.FORWARDREF) + self.assertIs(fr, evaluated) + + fr = ForwardRef("set[undefined]") + evaluated = fr.evaluate(format=Format.FORWARDREF) + self.assertEqual( + evaluated, + set[support.EqualToForwardRef("undefined")], + ) + + fr = ForwardRef("a + b") + self.assertEqual( + fr.evaluate(format=Format.FORWARDREF), + support.EqualToForwardRef("a + b"), + ) + self.assertEqual( + fr.evaluate(format=Format.FORWARDREF, locals={"a": 1, "b": 2}), + 3, + ) + + fr = ForwardRef('"a" + 1') + self.assertEqual( + fr.evaluate(format=Format.FORWARDREF), + support.EqualToForwardRef('"a" + 1'), + ) + + def test_evaluate_notimplemented_format(self): + class C: + x: alias + + fwdref = get_annotations(C, format=Format.FORWARDREF)["x"] + + with self.assertRaises(NotImplementedError): + fwdref.evaluate(format=Format.VALUE_WITH_FAKE_GLOBALS) + + with self.assertRaises(NotImplementedError): + # Some other unsupported value + fwdref.evaluate(format=7) + + def test_evaluate_with_type_params(self): + class Gen[T]: + alias = int + + with self.assertRaises(NameError): + ForwardRef("T").evaluate() + with self.assertRaises(NameError): + ForwardRef("T").evaluate(type_params=()) + with self.assertRaises(NameError): + ForwardRef("T").evaluate(owner=int) + + (T,) = Gen.__type_params__ + self.assertIs(ForwardRef("T").evaluate(type_params=Gen.__type_params__), T) + self.assertIs(ForwardRef("T").evaluate(owner=Gen), T) + + with self.assertRaises(NameError): + ForwardRef("alias").evaluate(type_params=Gen.__type_params__) + self.assertIs(ForwardRef("alias").evaluate(owner=Gen), int) + # If you pass custom locals, we don't look at the owner's locals + with self.assertRaises(NameError): + ForwardRef("alias").evaluate(owner=Gen, locals={}) + # But if the name exists in the locals, it works + self.assertIs( + ForwardRef("alias").evaluate(owner=Gen, locals={"alias": str}), str + ) + + def test_evaluate_with_type_params_and_scope_conflict(self): + for is_class in (False, True): + with self.subTest(is_class=is_class): + fwdref1 = ForwardRef("TypeParamsAlias1", owner=TypeParamsSample, is_class=is_class) + fwdref2 = ForwardRef("TypeParamsAlias2", owner=TypeParamsSample, is_class=is_class) + + self.assertIs( + fwdref1.evaluate(), + TypeParamsSample.__type_params__[0], + ) + self.assertIs( + fwdref2.evaluate(), + TypeParamsSample.TypeParamsAlias2, + ) + + def test_fwdref_with_module(self): + self.assertIs(ForwardRef("Format", module="annotationlib").evaluate(), Format) + self.assertIs( + ForwardRef("Counter", module="collections").evaluate(), collections.Counter + ) + self.assertEqual( + ForwardRef("Counter[int]", module="collections").evaluate(), + collections.Counter[int], + ) + + with self.assertRaises(NameError): + # If globals are passed explicitly, we don't look at the module dict + ForwardRef("Format", module="annotationlib").evaluate(globals={}) + + def test_fwdref_to_builtin(self): + self.assertIs(ForwardRef("int").evaluate(), int) + self.assertIs(ForwardRef("int", module="collections").evaluate(), int) + self.assertIs(ForwardRef("int", owner=str).evaluate(), int) + + # builtins are still searched with explicit globals + self.assertIs(ForwardRef("int").evaluate(globals={}), int) + + # explicit values in globals have precedence + obj = object() + self.assertIs(ForwardRef("int").evaluate(globals={"int": obj}), obj) + + def test_fwdref_value_is_not_cached(self): + fr = ForwardRef("hello") + with self.assertRaises(NameError): + fr.evaluate() + self.assertIs(fr.evaluate(globals={"hello": str}), str) + with self.assertRaises(NameError): + fr.evaluate() + + def test_fwdref_with_owner(self): + self.assertEqual( + ForwardRef("Counter[int]", owner=collections).evaluate(), + collections.Counter[int], + ) + + def test_name_lookup_without_eval(self): + # test the codepath where we look up simple names directly in the + # namespaces without going through eval() + self.assertIs(ForwardRef("int").evaluate(), int) + self.assertIs(ForwardRef("int").evaluate(locals={"int": str}), str) + self.assertIs( + ForwardRef("int").evaluate(locals={"int": float}, globals={"int": str}), + float, + ) + self.assertIs(ForwardRef("int").evaluate(globals={"int": str}), str) + with support.swap_attr(builtins, "int", dict): + self.assertIs(ForwardRef("int").evaluate(), dict) + + with self.assertRaises(NameError, msg="name 'doesntexist' is not defined") as exc: + ForwardRef("doesntexist").evaluate() + + self.assertEqual(exc.exception.name, "doesntexist") + + def test_evaluate_undefined_generic(self): + # Test the codepath where have to eval() with undefined variables. + class C: + x: alias[int, undef] + + generic = get_annotations(C, format=Format.FORWARDREF)["x"].evaluate( + format=Format.FORWARDREF, + globals={"alias": dict} + ) + self.assertNotIsInstance(generic, ForwardRef) + self.assertIs(generic.__origin__, dict) + self.assertEqual(len(generic.__args__), 2) + self.assertIs(generic.__args__[0], int) + self.assertIsInstance(generic.__args__[1], ForwardRef) + + generic = get_annotations(C, format=Format.FORWARDREF)["x"].evaluate( + format=Format.FORWARDREF, + globals={"alias": Union}, + locals={"alias": dict} + ) + self.assertNotIsInstance(generic, ForwardRef) + self.assertIs(generic.__origin__, dict) + self.assertEqual(len(generic.__args__), 2) + self.assertIs(generic.__args__[0], int) + self.assertIsInstance(generic.__args__[1], ForwardRef) + + def test_fwdref_invalid_syntax(self): + fr = ForwardRef("if") + with self.assertRaises(SyntaxError): + fr.evaluate() + fr = ForwardRef("1+") + with self.assertRaises(SyntaxError): + fr.evaluate() + + def test_re_evaluate_generics(self): + global global_alias + + # If we've already run this test before, + # ensure the variable is still undefined + if "global_alias" in globals(): + del global_alias + + class C: + x: global_alias[int] + + # Evaluate the ForwardRef once + evaluated = get_annotations(C, format=Format.FORWARDREF)["x"].evaluate( + format=Format.FORWARDREF + ) + + # Now define the global and ensure that the ForwardRef evaluates + global_alias = list + self.assertEqual(evaluated.evaluate(), list[int]) + + def test_fwdref_evaluate_argument_mutation(self): + class C[T]: + nonlocal alias + x: alias[T] + + # Mutable arguments + globals_ = globals() + globals_copy = globals_.copy() + locals_ = locals() + locals_copy = locals_.copy() + + # Evaluate the ForwardRef, ensuring we use __cell__ and type params + get_annotations(C, format=Format.FORWARDREF)["x"].evaluate( + globals=globals_, + locals=locals_, + type_params=C.__type_params__, + format=Format.FORWARDREF, + ) + + # Check if the passed in mutable arguments equal the originals + self.assertEqual(globals_, globals_copy) + self.assertEqual(locals_, locals_copy) + + alias = list + + def test_fwdref_final_class(self): + with self.assertRaises(TypeError): + class C(ForwardRef): + pass + + +class TestAnnotationLib(unittest.TestCase): + def test__all__(self): + support.check__all__(self, annotationlib) + + @support.cpython_only + def test_lazy_imports(self): + import_helper.ensure_lazy_imports( + "annotationlib", + { + "typing", + "warnings", + }, + ) diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 9b8179ef969..f48fb765bb3 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -3307,12 +3307,11 @@ def test_help_subparser_all_mutually_exclusive_group_members_suppressed(self): ''' self.assertEqual(cmd_foo.format_help(), textwrap.dedent(expected)) - def test_empty_group(self): + def test_usage_empty_group(self): # See issue 26952 - parser = argparse.ArgumentParser() + parser = ErrorRaisingArgumentParser(prog='PROG') group = parser.add_mutually_exclusive_group() - with self.assertRaises(ValueError): - parser.parse_args(['-h']) + self.assertEqual(parser.format_usage(), 'usage: PROG [-h]\n') def test_nested_mutex_groups(self): parser = argparse.ArgumentParser(prog='PROG') @@ -3580,25 +3579,29 @@ def get_parser(self, required): group.add_argument('-b', action='store_true', help='b help') parser.add_argument('-y', action='store_true', help='y help') group.add_argument('-c', action='store_true', help='c help') + parser.add_argument('-z', action='store_true', help='z help') return parser failures = ['-a -b', '-b -c', '-a -c', '-a -b -c'] successes = [ - ('-a', NS(a=True, b=False, c=False, x=False, y=False)), - ('-b', NS(a=False, b=True, c=False, x=False, y=False)), - ('-c', NS(a=False, b=False, c=True, x=False, y=False)), - ('-a -x', NS(a=True, b=False, c=False, x=True, y=False)), - ('-y -b', NS(a=False, b=True, c=False, x=False, y=True)), - ('-x -y -c', NS(a=False, b=False, c=True, x=True, y=True)), + ('-a', NS(a=True, b=False, c=False, x=False, y=False, z=False)), + ('-b', NS(a=False, b=True, c=False, x=False, y=False, z=False)), + ('-c', NS(a=False, b=False, c=True, x=False, y=False, z=False)), + ('-a -x', NS(a=True, b=False, c=False, x=True, y=False, z=False)), + ('-y -b', NS(a=False, b=True, c=False, x=False, y=True, z=False)), + ('-x -y -c', NS(a=False, b=False, c=True, x=True, y=True, z=False)), ] successes_when_not_required = [ - ('', NS(a=False, b=False, c=False, x=False, y=False)), - ('-x', NS(a=False, b=False, c=False, x=True, y=False)), - ('-y', NS(a=False, b=False, c=False, x=False, y=True)), + ('', NS(a=False, b=False, c=False, x=False, y=False, z=False)), + ('-x', NS(a=False, b=False, c=False, x=True, y=False, z=False)), + ('-y', NS(a=False, b=False, c=False, x=False, y=True, z=False)), ] - usage_when_required = usage_when_not_required = '''\ - usage: PROG [-h] [-x] [-a] [-b] [-y] [-c] + usage_when_not_required = '''\ + usage: PROG [-h] [-x] [-a | -b | -c] [-y] [-z] + ''' + usage_when_required = '''\ + usage: PROG [-h] [-x] (-a | -b | -c) [-y] [-z] ''' help = '''\ @@ -3609,6 +3612,7 @@ def get_parser(self, required): -b b help -y y help -c c help + -z z help ''' @@ -3662,23 +3666,27 @@ def get_parser(self, required): group.add_argument('a', nargs='?', help='a help') group.add_argument('-b', action='store_true', help='b help') group.add_argument('-c', action='store_true', help='c help') + parser.add_argument('-z', action='store_true', help='z help') return parser failures = ['X A -b', '-b -c', '-c X A'] successes = [ - ('X A', NS(a='A', b=False, c=False, x='X', y=False)), - ('X -b', NS(a=None, b=True, c=False, x='X', y=False)), - ('X -c', NS(a=None, b=False, c=True, x='X', y=False)), - ('X A -y', NS(a='A', b=False, c=False, x='X', y=True)), - ('X -y -b', NS(a=None, b=True, c=False, x='X', y=True)), + ('X A', NS(a='A', b=False, c=False, x='X', y=False, z=False)), + ('X -b', NS(a=None, b=True, c=False, x='X', y=False, z=False)), + ('X -c', NS(a=None, b=False, c=True, x='X', y=False, z=False)), + ('X A -y', NS(a='A', b=False, c=False, x='X', y=True, z=False)), + ('X -y -b', NS(a=None, b=True, c=False, x='X', y=True, z=False)), ] successes_when_not_required = [ - ('X', NS(a=None, b=False, c=False, x='X', y=False)), - ('X -y', NS(a=None, b=False, c=False, x='X', y=True)), + ('X', NS(a=None, b=False, c=False, x='X', y=False, z=False)), + ('X -y', NS(a=None, b=False, c=False, x='X', y=True, z=False)), ] - usage_when_required = usage_when_not_required = '''\ - usage: PROG [-h] [-y] [-b] [-c] x [a] + usage_when_not_required = '''\ + usage: PROG [-h] [-y] [-z] x [-b | -c | a] + ''' + usage_when_required = '''\ + usage: PROG [-h] [-y] [-z] x (-b | -c | a) ''' help = '''\ @@ -3691,6 +3699,7 @@ def get_parser(self, required): -y y help -b b help -c c help + -z z help ''' @@ -4885,6 +4894,25 @@ def test_long_mutex_groups_wrap(self): ''') self.assertEqual(parser.format_usage(), usage) + def test_mutex_groups_with_mixed_optionals_positionals_wrap(self): + # https://github.com/python/cpython/issues/75949 + # Mutually exclusive groups containing both optionals and positionals + # should preserve pipe separators when the usage line wraps. + parser = argparse.ArgumentParser(prog='PROG') + g = parser.add_mutually_exclusive_group() + g.add_argument('-v', '--verbose', action='store_true') + g.add_argument('-q', '--quiet', action='store_true') + g.add_argument('-x', '--extra-long-option-name', nargs='?') + g.add_argument('-y', '--yet-another-long-option', nargs='?') + g.add_argument('positional', nargs='?') + + usage = textwrap.dedent('''\ + usage: PROG [-h] + [-v | -q | -x [EXTRA_LONG_OPTION_NAME] | + -y [YET_ANOTHER_LONG_OPTION] | positional] + ''') + self.assertEqual(parser.format_usage(), usage) + class TestHelpVariableExpansion(HelpTestCase): """Test that variables are expanded properly in help messages""" @@ -5300,6 +5328,7 @@ class TestHelpArgumentDefaults(HelpTestCase): argument_signatures = [ Sig('--foo', help='foo help - oh and by the way, %(default)s'), Sig('--bar', action='store_true', help='bar help'), + Sig('--required', required=True, help='some help'), Sig('--taz', action=argparse.BooleanOptionalAction, help='Whether to taz it', default=True), Sig('--corge', action=argparse.BooleanOptionalAction, @@ -5313,8 +5342,8 @@ class TestHelpArgumentDefaults(HelpTestCase): [Sig('--baz', type=int, default=42, help='baz help')]), ] usage = '''\ - usage: PROG [-h] [--foo FOO] [--bar] [--taz | --no-taz] [--corge | --no-corge] - [--quux QUUX] [--baz BAZ] + usage: PROG [-h] [--foo FOO] [--bar] --required REQUIRED [--taz | --no-taz] + [--corge | --no-corge] [--quux QUUX] [--baz BAZ] spam [badger] ''' help = usage + '''\ @@ -5329,6 +5358,7 @@ class TestHelpArgumentDefaults(HelpTestCase): -h, --help show this help message and exit --foo FOO foo help - oh and by the way, None --bar bar help (default: False) + --required REQUIRED some help --taz, --no-taz Whether to taz it (default: True) --corge, --no-corge Whether to corge it --quux QUUX Set the quux (default: 42) @@ -7256,7 +7286,28 @@ def test_argparse_color(self): ), ) - def test_argparse_color_usage(self): + def test_argparse_color_mutually_exclusive_group_usage(self): + parser = argparse.ArgumentParser(color=True, prog="PROG") + group = parser.add_mutually_exclusive_group() + group.add_argument('--foo', action='store_true', help='FOO') + group.add_argument('--spam', help='SPAM') + group.add_argument('badger', nargs='*', help='BADGER') + + prog = self.theme.prog + heading = self.theme.heading + long = self.theme.summary_long_option + short = self.theme.summary_short_option + label = self.theme.summary_label + pos = self.theme.summary_action + reset = self.theme.reset + + self.assertEqual(parser.format_usage(), + f"{heading}usage: {reset}{prog}PROG{reset} [{short}-h{reset}] " + f"[{long}--foo{reset} | " + f"{long}--spam {label}SPAM{reset} | " + f"{pos}badger ...{reset}]\n") + + def test_argparse_color_custom_usage(self): # Arrange parser = argparse.ArgumentParser( add_help=False, diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py index 0c20e27cfda..db09e50e8f4 100644 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -8,16 +8,20 @@ from test.support import import_helper from test.support import os_helper from test.support import _2G +from test.support import subTests import weakref import pickle import operator import struct import sys +import warnings import array from array import _array_reconstructor as array_reconstructor -sizeof_wchar = array.array('u').itemsize +with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + sizeof_wchar = array.array('u').itemsize class ArraySubclass(array.array): @@ -27,7 +31,7 @@ class ArraySubclassWithKwargs(array.array): def __init__(self, typecode, newarg=None): array.array.__init__(self) -typecodes = 'ubBhHiIlLfdqQ' +typecodes = 'uwbBhHiIlLfdqQ' class MiscTest(unittest.TestCase): @@ -93,8 +97,17 @@ def test_empty(self): UTF32_LE = 20 UTF32_BE = 21 + class ArrayReconstructorTest(unittest.TestCase): + def setUp(self): + self.enterContext(warnings.catch_warnings()) + warnings.filterwarnings( + "ignore", + message="The 'u' type code is deprecated and " + "will be removed in Python 3.16", + category=DeprecationWarning) + def test_error(self): self.assertRaises(TypeError, array_reconstructor, "", "b", 0, b"") @@ -176,23 +189,23 @@ def test_numbers(self): self.assertEqual(a, b, msg="{0!r} != {1!r}; testcase={2!r}".format(a, b, testcase)) - # TODO: RUSTPYTHON - requires UTF-32 encoding support in codecs and proper array reconstructor implementation - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' def test_unicode(self): teststr = "Bonne Journ\xe9e \U0002030a\U00020347" testcases = ( (UTF16_LE, "UTF-16-LE"), (UTF16_BE, "UTF-16-BE"), - (UTF32_LE, "UTF-32-LE"), # TODO: RUSTPYTHON - (UTF32_BE, "UTF-32-BE") # TODO: RUSTPYTHON + (UTF32_LE, "UTF-32-LE"), + (UTF32_BE, "UTF-32-BE") ) for testcase in testcases: mformat_code, encoding = testcase - a = array.array('u', teststr) - b = array_reconstructor( - array.array, 'u', mformat_code, teststr.encode(encoding)) - self.assertEqual(a, b, - msg="{0!r} != {1!r}; testcase={2!r}".format(a, b, testcase)) + for c in 'uw': + a = array.array(c, teststr) + b = array_reconstructor( + array.array, c, mformat_code, teststr.encode(encoding)) + self.assertEqual(a, b, + msg="{0!r} != {1!r}; testcase={2!r}".format(a, b, testcase)) class BaseTest: @@ -204,6 +217,14 @@ class BaseTest: # outside: An entry that is not in example # minitemsize: the minimum guaranteed itemsize + def setUp(self): + self.enterContext(warnings.catch_warnings()) + warnings.filterwarnings( + "ignore", + message="The 'u' type code is deprecated and " + "will be removed in Python 3.16", + category=DeprecationWarning) + def assertEntryEqual(self, entry1, entry2): self.assertEqual(entry1, entry2) @@ -236,7 +257,7 @@ def test_buffer_info(self): self.assertEqual(bi[1], len(a)) def test_byteswap(self): - if self.typecode == 'u': + if self.typecode in ('u', 'w'): example = '\U00100100' else: example = self.example @@ -357,8 +378,6 @@ def test_reverse_iterator(self): self.assertEqual(list(a), list(self.example)) self.assertEqual(list(reversed(a)), list(iter(a))[::-1]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_reverse_iterator_picking(self): orig = array.array(self.typecode, self.example) data = list(orig) @@ -997,6 +1016,29 @@ def test_pop(self): array.array(self.typecode, self.example[3:]+self.example[:-1]) ) + def test_clear(self): + a = array.array(self.typecode, self.example) + with self.assertRaises(TypeError): + a.clear(42) + a.clear() + self.assertEqual(len(a), 0) + self.assertEqual(a.typecode, self.typecode) + + a = array.array(self.typecode) + a.clear() + self.assertEqual(len(a), 0) + self.assertEqual(a.typecode, self.typecode) + + a = array.array(self.typecode, self.example) + a.clear() + a.append(self.example[2]) + a.append(self.example[3]) + self.assertEqual(a, array.array(self.typecode, self.example[2:4])) + + with memoryview(a): + with self.assertRaises(BufferError): + a.clear() + def test_reverse(self): a = array.array(self.typecode, self.example) self.assertRaises(TypeError, a.reverse, 42) @@ -1083,7 +1125,7 @@ def test_buffer(self): self.assertEqual(m.tobytes(), expected) self.assertRaises(BufferError, a.frombytes, a.tobytes()) self.assertEqual(m.tobytes(), expected) - if self.typecode == 'u': + if self.typecode in ('u', 'w'): self.assertRaises(BufferError, a.fromunicode, a.tounicode()) self.assertEqual(m.tobytes(), expected) self.assertRaises(BufferError, operator.imul, a, 2) @@ -1138,17 +1180,19 @@ def test_sizeof_without_buffer(self): basesize = support.calcvobjsize('Pn2Pi') support.check_sizeof(self, a, basesize) + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' def test_initialize_with_unicode(self): - if self.typecode != 'u': + if self.typecode not in ('u', 'w'): with self.assertRaises(TypeError) as cm: a = array.array(self.typecode, 'foo') self.assertIn("cannot use a str", str(cm.exception)) with self.assertRaises(TypeError) as cm: - a = array.array(self.typecode, array.array('u', 'foo')) + a = array.array(self.typecode, array.array('w', 'foo')) self.assertIn("cannot use a unicode array", str(cm.exception)) else: a = array.array(self.typecode, "foo") a = array.array(self.typecode, array.array('u', 'foo')) + a = array.array(self.typecode, array.array('w', 'foo')) @support.cpython_only def test_obsolete_write_lock(self): @@ -1156,8 +1200,7 @@ def test_obsolete_write_lock(self): a = array.array('B', b"") self.assertRaises(BufferError, _testcapi.getbuffer_with_null_view, a) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_free_after_iterating(self): support.check_free_after_iterating(self, iter, array.array, (self.typecode,)) @@ -1177,40 +1220,255 @@ class UnicodeTest(StringTest, unittest.TestCase): smallerexample = '\x01\u263a\x00\ufefe' biggerexample = '\x01\u263a\x01\ufeff' outside = str('\x33') - minitemsize = 2 + minitemsize = sizeof_wchar def test_unicode(self): self.assertRaises(TypeError, array.array, 'b', 'foo') - a = array.array('u', '\xa0\xc2\u1234') + a = array.array(self.typecode, '\xa0\xc2\u1234') a.fromunicode(' ') a.fromunicode('') a.fromunicode('') a.fromunicode('\x11abc\xff\u1234') s = a.tounicode() self.assertEqual(s, '\xa0\xc2\u1234 \x11abc\xff\u1234') - self.assertEqual(a.itemsize, sizeof_wchar) + self.assertEqual(a.itemsize, self.minitemsize) s = '\x00="\'a\\b\x80\xff\u0000\u0001\u1234' - a = array.array('u', s) + a = array.array(self.typecode, s) self.assertEqual( repr(a), - "array('u', '\\x00=\"\\'a\\\\b\\x80\xff\\x00\\x01\u1234')") + f"array('{self.typecode}', '\\x00=\"\\'a\\\\b\\x80\xff\\x00\\x01\u1234')") self.assertRaises(TypeError, a.fromunicode) def test_issue17223(self): - # this used to crash - if sizeof_wchar == 4: - # U+FFFFFFFF is an invalid code point in Unicode 6.0 - invalid_str = b'\xff\xff\xff\xff' - else: + if self.typecode == 'u' and sizeof_wchar == 2: # PyUnicode_FromUnicode() cannot fail with 16-bit wchar_t self.skipTest("specific to 32-bit wchar_t") - a = array.array('u', invalid_str) + + # this used to crash + # U+FFFFFFFF is an invalid code point in Unicode 6.0 + invalid_str = b'\xff\xff\xff\xff' + + a = array.array(self.typecode, invalid_str) self.assertRaises(ValueError, a.tounicode) self.assertRaises(ValueError, str, a) + def test_typecode_u_deprecation(self): + with self.assertWarns(DeprecationWarning): + array.array("u") + + def test_empty_string_mem_leak_gh140474(self): + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + for _ in range(1000): + a = array.array('u', '') + self.assertEqual(len(a), 0) + self.assertEqual(a.typecode, 'u') + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_add(self): + return super().test_add() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_extend(self): + return super().test_extend() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_iadd(self): + return super().test_iadd() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_setiadd(self): + return super().test_setiadd() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_setslice(self): + return super().test_setslice() + + +class UCS4Test(UnicodeTest): + typecode = 'w' + minitemsize = 4 + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_buffer(self): + return super().test_buffer() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_buffer_info(self): + return super().test_buffer_info() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_byteswap(self): + return super().test_byteswap() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_clear(self): + return super().test_clear() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_cmp(self): + return super().test_cmp() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_constructor(self): + return super().test_constructor() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_constructor_with_iterable_argument(self): + return super().test_constructor_with_iterable_argument() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_copy(self): + return super().test_copy() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_count(self): + return super().test_count() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_coveritertraverse(self): + return super().test_coveritertraverse() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_deepcopy(self): + return super().test_deepcopy() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_delitem(self): + return super().test_delitem() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_exhausted_iterator(self): + return super().test_exhausted_iterator() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_exhausted_reverse_iterator(self): + return super().test_exhausted_reverse_iterator() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_extended_getslice(self): + return super().test_extended_getslice() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_extended_set_del_slice(self): + return super().test_extended_set_del_slice() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_filewrite(self): + return super().test_filewrite() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_fromarray(self): + return super().test_fromarray() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_fromfile_ioerror(self): + return super().test_fromfile_ioerror() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_getitem(self): + return super().test_getitem() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_getslice(self): + return super().test_getslice() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_imul(self): + return super().test_imul() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_index(self): + return super().test_index() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_insert(self): + return super().test_insert() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_issue17223(self): + return super().test_issue17223() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_iterator_pickle(self): + return super().test_iterator_pickle() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_len(self): + return super().test_len() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_mul(self): + return super().test_mul() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_pickle(self): + return super().test_pickle() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_pickle_for_empty_array(self): + return super().test_pickle_for_empty_array() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_pop(self): + return super().test_pop() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_reduce_ex(self): + return super().test_reduce_ex() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_remove(self): + return super().test_remove() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_repr(self): + return super().test_repr() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_reverse(self): + return super().test_reverse() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_reverse_iterator(self): + return super().test_reverse_iterator() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_reverse_iterator_picking(self): + return super().test_reverse_iterator_picking() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_setitem(self): + return super().test_setitem() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_str(self): + return super().test_str() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_tofrombytes(self): + return super().test_tofrombytes() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_tofromfile(self): + return super().test_tofromfile() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_tofromlist(self): + return super().test_tofromlist() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_unicode(self): + return super().test_unicode() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_weakref(self): + return super().test_weakref() + + class NumberTest(BaseTest): def test_extslice(self): @@ -1442,8 +1700,8 @@ def test_byteswap(self): if a.itemsize==1: self.assertEqual(a, b) else: - # On alphas treating the byte swapped bit patters as - # floats/doubles results in floating point exceptions + # On alphas treating the byte swapped bit patterns as + # floats/doubles results in floating-point exceptions # => compare the 8bit string values instead self.assertNotEqual(a.tobytes(), b.tobytes()) b.byteswap() @@ -1615,5 +1873,55 @@ def test_tolist(self, size): self.assertEqual(ls[:8], list(example[:8])) self.assertEqual(ls[-8:], list(example[-8:])) + def test_gh_128961(self): + a = array.array('i') + it = iter(a) + list(it) + it.__setstate__(0) + self.assertRaises(StopIteration, next, it) + + # Tests for NULL pointer dereference in array.__setitem__ + # when the index conversion mutates the array. + # See: https://github.com/python/cpython/issues/142555. + + @unittest.skip("TODO: RUSTPYTHON; Hangs") + @subTests("dtype", ["b", "B", "h", "H", "i", "l", "q", "I", "L", "Q"]) + def test_setitem_use_after_clear_with_int_data(self, dtype): + victim = array.array(dtype, list(range(64))) + + class Index: + def __index__(self): + victim.clear() + return 0 + + self.assertRaises(IndexError, victim.__setitem__, 1, Index()) + self.assertEqual(len(victim), 0) + + @unittest.skip("TODO: RUSTPYTHON; Hangs") + def test_setitem_use_after_shrink_with_int_data(self): + victim = array.array('b', [1, 2, 3]) + + class Index: + def __index__(self): + victim.pop() + victim.pop() + return 0 + + self.assertRaises(IndexError, victim.__setitem__, 1, Index()) + + @unittest.skip("TODO: RUSTPYTHON; Hangs") + @subTests("dtype", ["f", "d"]) + def test_setitem_use_after_clear_with_float_data(self, dtype): + victim = array.array(dtype, [1.0, 2.0, 3.0]) + + class Float: + def __float__(self): + victim.clear() + return 0.0 + + self.assertRaises(IndexError, victim.__setitem__, 1, Float()) + self.assertEqual(len(victim), 0) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_ast/data/ast_repr.txt b/Lib/test/test_ast/data/ast_repr.txt new file mode 100644 index 00000000000..1c1985519cd --- /dev/null +++ b/Lib/test/test_ast/data/ast_repr.txt @@ -0,0 +1,214 @@ +Module(body=[Expr(value=Constant(value='module docstring', kind=None))], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Expr(value=Constant(...))], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[arg(...)], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[arg(...)], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[Constant(...)]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=arg(...), kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=arg(...), kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=arg(...), kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=arg(...), kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=arg(...), defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[arg(...), ..., arg(...)], vararg=arg(...), kwonlyargs=[arg(...)], kw_defaults=[Constant(...)], kwarg=arg(...), defaults=[Constant(...), ..., Dict(...)]), body=[Expr(value=Constant(...))], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=Subscript(value=Name(...), slice=Tuple(...), ctx=Load(...)), type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=Subscript(value=Name(...), slice=Tuple(...), ctx=Load(...)), type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=Subscript(value=Name(...), slice=Tuple(...), ctx=Load(...)), type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[ClassDef(name='C', bases=[], keywords=[], body=[Pass()], decorator_list=[], type_params=[])], type_ignores=[]) +Module(body=[ClassDef(name='C', bases=[], keywords=[], body=[Expr(value=Constant(...))], decorator_list=[], type_params=[])], type_ignores=[]) +Module(body=[ClassDef(name='C', bases=[Name(id='object', ctx=Load(...))], keywords=[], body=[Pass()], decorator_list=[], type_params=[])], type_ignores=[]) +Module(body=[ClassDef(name='C', bases=[Name(id='A', ctx=Load(...)), Name(id='B', ctx=Load(...))], keywords=[], body=[Pass()], decorator_list=[], type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Return(value=Constant(...))], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Return(value=None)], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[Delete(targets=[Name(id='v', ctx=Del(...))])], type_ignores=[]) +Module(body=[Assign(targets=[Name(id='v', ctx=Store(...))], value=Constant(value=1, kind=None), type_comment=None)], type_ignores=[]) +Module(body=[Assign(targets=[Tuple(elts=[Name(...), Name(...)], ctx=Store(...))], value=Name(id='c', ctx=Load(...)), type_comment=None)], type_ignores=[]) +Module(body=[Assign(targets=[Tuple(elts=[Name(...), Name(...)], ctx=Store(...))], value=Name(id='c', ctx=Load(...)), type_comment=None)], type_ignores=[]) +Module(body=[Assign(targets=[List(elts=[Name(...), Name(...)], ctx=Store(...))], value=Name(id='c', ctx=Load(...)), type_comment=None)], type_ignores=[]) +Module(body=[Assign(targets=[Subscript(value=Name(...), slice=Name(...), ctx=Store(...))], value=Name(id='c', ctx=Load(...)), type_comment=None)], type_ignores=[]) +Module(body=[AnnAssign(target=Name(id='x', ctx=Store(...)), annotation=Subscript(value=Name(...), slice=Tuple(...), ctx=Load(...)), value=None, simple=1)], type_ignores=[]) +Module(body=[AnnAssign(target=Name(id='x', ctx=Store(...)), annotation=Subscript(value=Name(...), slice=Tuple(...), ctx=Load(...)), value=None, simple=1)], type_ignores=[]) +Module(body=[AnnAssign(target=Name(id='x', ctx=Store(...)), annotation=Subscript(value=Name(...), slice=Tuple(...), ctx=Load(...)), value=None, simple=1)], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=Add(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=Sub(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=Mult(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=MatMult(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=Div(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=Mod(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=Pow(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=LShift(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=RShift(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=BitOr(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=BitXor(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=BitAnd(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=FloorDiv(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[For(target=Name(id='v', ctx=Store(...)), iter=Name(id='v', ctx=Load(...)), body=[Pass()], orelse=[], type_comment=None)], type_ignores=[]) +Module(body=[For(target=Name(id='v', ctx=Store(...)), iter=Name(id='v', ctx=Load(...)), body=[Pass()], orelse=[Pass()], type_comment=None)], type_ignores=[]) +Module(body=[While(test=Name(id='v', ctx=Load(...)), body=[Pass()], orelse=[])], type_ignores=[]) +Module(body=[While(test=Name(id='v', ctx=Load(...)), body=[Pass()], orelse=[Pass()])], type_ignores=[]) +Module(body=[If(test=Name(id='v', ctx=Load(...)), body=[Pass()], orelse=[])], type_ignores=[]) +Module(body=[If(test=Name(id='a', ctx=Load(...)), body=[Pass()], orelse=[If(test=Name(...), body=[Pass(...)], orelse=[])])], type_ignores=[]) +Module(body=[If(test=Name(id='a', ctx=Load(...)), body=[Pass()], orelse=[Pass()])], type_ignores=[]) +Module(body=[If(test=Name(id='a', ctx=Load(...)), body=[Pass()], orelse=[If(test=Name(...), body=[Pass(...)], orelse=[Pass(...)])])], type_ignores=[]) +Module(body=[If(test=Name(id='a', ctx=Load(...)), body=[Pass()], orelse=[If(test=Name(...), body=[Pass(...)], orelse=[If(...)])])], type_ignores=[]) +Module(body=[With(items=[withitem(context_expr=Name(...), optional_vars=None)], body=[Pass()], type_comment=None)], type_ignores=[]) +Module(body=[With(items=[withitem(context_expr=Name(...), optional_vars=None), withitem(context_expr=Name(...), optional_vars=None)], body=[Pass()], type_comment=None)], type_ignores=[]) +Module(body=[With(items=[withitem(context_expr=Name(...), optional_vars=Name(...))], body=[Pass()], type_comment=None)], type_ignores=[]) +Module(body=[With(items=[withitem(context_expr=Name(...), optional_vars=Name(...)), withitem(context_expr=Name(...), optional_vars=Name(...))], body=[Pass()], type_comment=None)], type_ignores=[]) +Module(body=[With(items=[withitem(context_expr=Name(...), optional_vars=Name(...))], body=[Pass()], type_comment=None)], type_ignores=[]) +Module(body=[With(items=[withitem(context_expr=Name(...), optional_vars=None), withitem(context_expr=Name(...), optional_vars=None)], body=[Pass()], type_comment=None)], type_ignores=[]) +Module(body=[Raise(exc=None, cause=None)], type_ignores=[]) +Module(body=[Raise(exc=Call(func=Name(...), args=[Constant(...)], keywords=[]), cause=None)], type_ignores=[]) +Module(body=[Raise(exc=Name(id='Exception', ctx=Load(...)), cause=None)], type_ignores=[]) +Module(body=[Raise(exc=Call(func=Name(...), args=[Constant(...)], keywords=[]), cause=Constant(value=None, kind=None))], type_ignores=[]) +Module(body=[Try(body=[Pass()], handlers=[ExceptHandler(type=Name(...), name=None, body=[Pass(...)])], orelse=[], finalbody=[])], type_ignores=[]) +Module(body=[Try(body=[Pass()], handlers=[ExceptHandler(type=Name(...), name='exc', body=[Pass(...)])], orelse=[], finalbody=[])], type_ignores=[]) +Module(body=[Try(body=[Pass()], handlers=[], orelse=[], finalbody=[Pass()])], type_ignores=[]) +Module(body=[TryStar(body=[Pass()], handlers=[ExceptHandler(type=Name(...), name=None, body=[Pass(...)])], orelse=[], finalbody=[])], type_ignores=[]) +Module(body=[TryStar(body=[Pass()], handlers=[ExceptHandler(type=Name(...), name='exc', body=[Pass(...)])], orelse=[], finalbody=[])], type_ignores=[]) +Module(body=[Try(body=[Pass()], handlers=[ExceptHandler(type=Name(...), name=None, body=[Pass(...)])], orelse=[Pass()], finalbody=[Pass()])], type_ignores=[]) +Module(body=[Try(body=[Pass()], handlers=[ExceptHandler(type=Name(...), name='exc', body=[Pass(...)])], orelse=[Pass()], finalbody=[Pass()])], type_ignores=[]) +Module(body=[TryStar(body=[Pass()], handlers=[ExceptHandler(type=Name(...), name='exc', body=[Pass(...)])], orelse=[Pass()], finalbody=[Pass()])], type_ignores=[]) +Module(body=[Assert(test=Name(id='v', ctx=Load(...)), msg=None)], type_ignores=[]) +Module(body=[Assert(test=Name(id='v', ctx=Load(...)), msg=Constant(value='message', kind=None))], type_ignores=[]) +Module(body=[Import(names=[alias(name='sys', asname=None)])], type_ignores=[]) +Module(body=[Import(names=[alias(name='foo', asname='bar')])], type_ignores=[]) +Module(body=[ImportFrom(module='sys', names=[alias(name='x', asname='y')], level=0)], type_ignores=[]) +Module(body=[ImportFrom(module='sys', names=[alias(name='v', asname=None)], level=0)], type_ignores=[]) +Module(body=[Global(names=['v'])], type_ignores=[]) +Module(body=[Expr(value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[Pass()], type_ignores=[]) +Module(body=[For(target=Name(id='v', ctx=Store(...)), iter=Name(id='v', ctx=Load(...)), body=[Break()], orelse=[], type_comment=None)], type_ignores=[]) +Module(body=[For(target=Name(id='v', ctx=Store(...)), iter=Name(id='v', ctx=Load(...)), body=[Continue()], orelse=[], type_comment=None)], type_ignores=[]) +Module(body=[For(target=Tuple(elts=[Name(...), Name(...)], ctx=Store(...)), iter=Name(id='c', ctx=Load(...)), body=[Pass()], orelse=[], type_comment=None)], type_ignores=[]) +Module(body=[For(target=Tuple(elts=[Name(...), Name(...)], ctx=Store(...)), iter=Name(id='c', ctx=Load(...)), body=[Pass()], orelse=[], type_comment=None)], type_ignores=[]) +Module(body=[For(target=List(elts=[Name(...), Name(...)], ctx=Store(...)), iter=Name(id='c', ctx=Load(...)), body=[Pass()], orelse=[], type_comment=None)], type_ignores=[]) +Module(body=[Expr(value=GeneratorExp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=DictComp(key=Name(...), value=Name(...), generators=[comprehension(...), comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=DictComp(key=Name(...), value=Name(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=SetComp(elt=Name(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=SetComp(elt=Name(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[AsyncFunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Expr(value=Constant(...)), Expr(value=Await(...))], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[AsyncFunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[AsyncFor(target=Name(...), iter=Name(...), body=[Expr(...)], orelse=[Expr(...)], type_comment=None)], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[AsyncFunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[AsyncWith(items=[withitem(...)], body=[Expr(...)], type_comment=None)], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[Expr(value=Dict(keys=[None, Constant(...)], values=[Dict(...), Constant(...)]))], type_ignores=[]) +Module(body=[Expr(value=Set(elts=[Starred(...), Constant(...)]))], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Expr(value=Yield(...))], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Expr(value=YieldFrom(...))], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[AsyncFunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Expr(value=ListComp(...))], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[Name(id='deco1', ctx=Load(...)), ..., Call(func=Name(...), args=[Constant(...)], keywords=[])], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[AsyncFunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[Name(id='deco1', ctx=Load(...)), ..., Call(func=Name(...), args=[Constant(...)], keywords=[])], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[ClassDef(name='C', bases=[], keywords=[], body=[Pass()], decorator_list=[Name(id='deco1', ctx=Load(...)), ..., Call(func=Name(...), args=[Constant(...)], keywords=[])], type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[Call(func=Name(...), args=[GeneratorExp(...)], keywords=[])], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[Attribute(value=Attribute(...), attr='c', ctx=Load(...))], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[Expr(value=NamedExpr(target=Name(...), value=Constant(...)))], type_ignores=[]) +Module(body=[If(test=NamedExpr(target=Name(...), value=Call(...)), body=[Pass()], orelse=[])], type_ignores=[]) +Module(body=[While(test=NamedExpr(target=Name(...), value=Call(...)), body=[Pass()], orelse=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[arg(...), ..., arg(...)], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[arg(...)], vararg=None, kwonlyargs=[arg(...), arg(...)], kw_defaults=[None, None], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[arg(...)], vararg=None, kwonlyargs=[arg(...), arg(...)], kw_defaults=[None, None], kwarg=arg(...), defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[Constant(...)]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[arg(...), arg(...)], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[Constant(...), ..., Constant(...)]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[arg(...)], vararg=None, kwonlyargs=[arg(...)], kw_defaults=[Constant(...)], kwarg=None, defaults=[Constant(...), Constant(...)]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[arg(...)], vararg=None, kwonlyargs=[arg(...)], kw_defaults=[None], kwarg=None, defaults=[Constant(...), Constant(...)]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[arg(...)], vararg=None, kwonlyargs=[arg(...)], kw_defaults=[Constant(...)], kwarg=arg(...), defaults=[Constant(...), Constant(...)]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[arg(...)], vararg=None, kwonlyargs=[arg(...)], kw_defaults=[None], kwarg=arg(...), defaults=[Constant(...), Constant(...)]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[TypeAlias(name=Name(id='X', ctx=Store(...)), type_params=[], value=Name(id='int', ctx=Load(...)))], type_ignores=[]) +Module(body=[TypeAlias(name=Name(id='X', ctx=Store(...)), type_params=[TypeVar(name='T', bound=None, default_value=None)], value=Name(id='int', ctx=Load(...)))], type_ignores=[]) +Module(body=[TypeAlias(name=Name(id='X', ctx=Store(...)), type_params=[TypeVar(name='T', bound=None, default_value=None), ..., ParamSpec(name='P', default_value=None)], value=Tuple(elts=[Name(...), ..., Name(...)], ctx=Load(...)))], type_ignores=[]) +Module(body=[TypeAlias(name=Name(id='X', ctx=Store(...)), type_params=[TypeVar(name='T', bound=Name(...), default_value=None), ..., ParamSpec(name='P', default_value=None)], value=Tuple(elts=[Name(...), ..., Name(...)], ctx=Load(...)))], type_ignores=[]) +Module(body=[TypeAlias(name=Name(id='X', ctx=Store(...)), type_params=[TypeVar(name='T', bound=Tuple(...), default_value=None), ..., ParamSpec(name='P', default_value=None)], value=Tuple(elts=[Name(...), ..., Name(...)], ctx=Load(...)))], type_ignores=[]) +Module(body=[TypeAlias(name=Name(id='X', ctx=Store(...)), type_params=[TypeVar(name='T', bound=Name(...), default_value=Constant(...)), ..., ParamSpec(name='P', default_value=Constant(...))], value=Tuple(elts=[Name(...), ..., Name(...)], ctx=Load(...)))], type_ignores=[]) +Module(body=[ClassDef(name='X', bases=[], keywords=[], body=[Pass()], decorator_list=[], type_params=[TypeVar(name='T', bound=None, default_value=None)])], type_ignores=[]) +Module(body=[ClassDef(name='X', bases=[], keywords=[], body=[Pass()], decorator_list=[], type_params=[TypeVar(name='T', bound=None, default_value=None), ..., ParamSpec(name='P', default_value=None)])], type_ignores=[]) +Module(body=[ClassDef(name='X', bases=[], keywords=[], body=[Pass()], decorator_list=[], type_params=[TypeVar(name='T', bound=Name(...), default_value=None), ..., ParamSpec(name='P', default_value=None)])], type_ignores=[]) +Module(body=[ClassDef(name='X', bases=[], keywords=[], body=[Pass()], decorator_list=[], type_params=[TypeVar(name='T', bound=Tuple(...), default_value=None), ..., ParamSpec(name='P', default_value=None)])], type_ignores=[]) +Module(body=[ClassDef(name='X', bases=[], keywords=[], body=[Pass()], decorator_list=[], type_params=[TypeVar(name='T', bound=Name(...), default_value=Constant(...)), ..., ParamSpec(name='P', default_value=Constant(...))])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[TypeVar(name='T', bound=None, default_value=None)])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[TypeVar(name='T', bound=None, default_value=None), ..., ParamSpec(name='P', default_value=None)])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[TypeVar(name='T', bound=Name(...), default_value=None), ..., ParamSpec(name='P', default_value=None)])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[TypeVar(name='T', bound=Tuple(...), default_value=None), ..., ParamSpec(name='P', default_value=None)])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[TypeVar(name='T', bound=Name(...), default_value=Constant(...)), ..., ParamSpec(name='P', default_value=Constant(...))])], type_ignores=[]) +Module(body=[Match(subject=Name(id='x', ctx=Load(...)), cases=[match_case(pattern=MatchValue(...), guard=None, body=[Pass(...)])])], type_ignores=[]) +Module(body=[Match(subject=Name(id='x', ctx=Load(...)), cases=[match_case(pattern=MatchValue(...), guard=None, body=[Pass(...)]), match_case(pattern=MatchAs(...), guard=None, body=[Pass(...)])])], type_ignores=[]) +Module(body=[Expr(value=Constant(value=None, kind=None))], type_ignores=[]) +Module(body=[Expr(value=Constant(value=True, kind=None))], type_ignores=[]) +Module(body=[Expr(value=Constant(value=False, kind=None))], type_ignores=[]) +Module(body=[Expr(value=BoolOp(op=And(...), values=[Name(...), Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=BoolOp(op=Or(...), values=[Name(...), Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=Add(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=Sub(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=Mult(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=Div(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=MatMult(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=FloorDiv(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=Pow(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=Mod(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=RShift(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=LShift(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=BitXor(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=BitOr(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=BitAnd(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=UnaryOp(op=Not(...), operand=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=UnaryOp(op=UAdd(...), operand=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=UnaryOp(op=USub(...), operand=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=UnaryOp(op=Invert(...), operand=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=Lambda(args=arguments(...), body=Constant(...)))], type_ignores=[]) +Module(body=[Expr(value=Dict(keys=[Constant(...)], values=[Constant(...)]))], type_ignores=[]) +Module(body=[Expr(value=Dict(keys=[], values=[]))], type_ignores=[]) +Module(body=[Expr(value=Set(elts=[Constant(...)]))], type_ignores=[]) +Module(body=[Expr(value=Dict(keys=[Constant(...)], values=[Constant(...)]))], type_ignores=[]) +Module(body=[Expr(value=List(elts=[Constant(...), Constant(...)], ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Tuple(elts=[Constant(...)], ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Set(elts=[Constant(...), Constant(...)]))], type_ignores=[]) +Module(body=[Expr(value=ListComp(elt=Name(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=GeneratorExp(elt=Name(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=SetComp(elt=Name(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=DictComp(key=Name(...), value=Name(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=ListComp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=ListComp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=ListComp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=SetComp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=SetComp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=SetComp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=GeneratorExp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=GeneratorExp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=GeneratorExp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=Compare(left=Constant(...), ops=[Lt(...), Lt(...)], comparators=[Constant(...), Constant(...)]))], type_ignores=[]) +Module(body=[Expr(value=Compare(left=Name(...), ops=[Eq(...)], comparators=[Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=Compare(left=Name(...), ops=[LtE(...)], comparators=[Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=Compare(left=Name(...), ops=[GtE(...)], comparators=[Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=Compare(left=Name(...), ops=[NotEq(...)], comparators=[Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=Compare(left=Name(...), ops=[Is(...)], comparators=[Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=Compare(left=Name(...), ops=[IsNot(...)], comparators=[Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=Compare(left=Name(...), ops=[In(...)], comparators=[Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=Compare(left=Name(...), ops=[NotIn(...)], comparators=[Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=Call(func=Name(...), args=[], keywords=[]))], type_ignores=[]) +Module(body=[Expr(value=Call(func=Name(...), args=[Constant(...), ..., Starred(...)], keywords=[keyword(...), keyword(...)]))], type_ignores=[]) +Module(body=[Expr(value=Call(func=Name(...), args=[Starred(...)], keywords=[]))], type_ignores=[]) +Module(body=[Expr(value=Call(func=Name(...), args=[GeneratorExp(...)], keywords=[]))], type_ignores=[]) +Module(body=[Expr(value=Constant(value=10, kind=None))], type_ignores=[]) +Module(body=[Expr(value=Constant(value=1j, kind=None))], type_ignores=[]) +Module(body=[Expr(value=Constant(value='string', kind=None))], type_ignores=[]) +Module(body=[Expr(value=Attribute(value=Name(...), attr='b', ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Subscript(value=Name(...), slice=Slice(...), ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Name(id='v', ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=List(elts=[Constant(...), ..., Constant(...)], ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=List(elts=[], ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Tuple(elts=[Constant(...), ..., Constant(...)], ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Tuple(elts=[Constant(...), ..., Constant(...)], ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Tuple(elts=[], ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Call(func=Attribute(...), args=[Subscript(...)], keywords=[]))], type_ignores=[]) +Module(body=[Expr(value=Subscript(value=List(...), slice=Slice(...), ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Subscript(value=List(...), slice=Slice(...), ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Subscript(value=List(...), slice=Slice(...), ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Subscript(value=List(...), slice=Slice(...), ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=IfExp(test=Name(...), body=Call(...), orelse=Call(...)))], type_ignores=[]) +Module(body=[Expr(value=JoinedStr(values=[FormattedValue(...)]))], type_ignores=[]) +Module(body=[Expr(value=JoinedStr(values=[FormattedValue(...)]))], type_ignores=[]) +Module(body=[Expr(value=JoinedStr(values=[FormattedValue(...)]))], type_ignores=[]) +Module(body=[Expr(value=JoinedStr(values=[Constant(...), ..., Constant(...)]))], type_ignores=[]) +Module(body=[Expr(value=TemplateStr(values=[Interpolation(...)]))], type_ignores=[]) +Module(body=[Expr(value=TemplateStr(values=[Interpolation(...)]))], type_ignores=[]) +Module(body=[Expr(value=TemplateStr(values=[Interpolation(...)]))], type_ignores=[]) +Module(body=[Expr(value=TemplateStr(values=[Interpolation(...)]))], type_ignores=[]) +Module(body=[Expr(value=TemplateStr(values=[Constant(...), ..., Constant(...)]))], type_ignores=[]) \ No newline at end of file diff --git a/Lib/test/test_ast/snippets.py b/Lib/test/test_ast/snippets.py index 28d32b2941f..b76f98901d2 100644 --- a/Lib/test/test_ast/snippets.py +++ b/Lib/test/test_ast/snippets.py @@ -364,6 +364,12 @@ "f'{a:.2f}'", "f'{a!r}'", "f'foo({a})'", + # TemplateStr and Interpolation + "t'{a}'", + "t'{a:.2f}'", + "t'{a!r}'", + "t'{a!r:.2f}'", + "t'foo({a})'", ] @@ -597,5 +603,10 @@ def main(): ('Expression', ('JoinedStr', (1, 0, 1, 10), [('FormattedValue', (1, 2, 1, 9), ('Name', (1, 3, 1, 4), 'a', ('Load',)), -1, ('JoinedStr', (1, 4, 1, 8), [('Constant', (1, 5, 1, 8), '.2f', None)]))])), ('Expression', ('JoinedStr', (1, 0, 1, 8), [('FormattedValue', (1, 2, 1, 7), ('Name', (1, 3, 1, 4), 'a', ('Load',)), 114, None)])), ('Expression', ('JoinedStr', (1, 0, 1, 11), [('Constant', (1, 2, 1, 6), 'foo(', None), ('FormattedValue', (1, 6, 1, 9), ('Name', (1, 7, 1, 8), 'a', ('Load',)), -1, None), ('Constant', (1, 9, 1, 10), ')', None)])), +('Expression', ('TemplateStr', (1, 0, 1, 6), [('Interpolation', (1, 2, 1, 5), ('Name', (1, 3, 1, 4), 'a', ('Load',)), 'a', -1, None)])), +('Expression', ('TemplateStr', (1, 0, 1, 10), [('Interpolation', (1, 2, 1, 9), ('Name', (1, 3, 1, 4), 'a', ('Load',)), 'a', -1, ('JoinedStr', (1, 4, 1, 8), [('Constant', (1, 5, 1, 8), '.2f', None)]))])), +('Expression', ('TemplateStr', (1, 0, 1, 8), [('Interpolation', (1, 2, 1, 7), ('Name', (1, 3, 1, 4), 'a', ('Load',)), 'a', 114, None)])), +('Expression', ('TemplateStr', (1, 0, 1, 12), [('Interpolation', (1, 2, 1, 11), ('Name', (1, 3, 1, 4), 'a', ('Load',)), 'a', 114, ('JoinedStr', (1, 6, 1, 10), [('Constant', (1, 7, 1, 10), '.2f', None)]))])), +('Expression', ('TemplateStr', (1, 0, 1, 11), [('Constant', (1, 2, 1, 6), 'foo(', None), ('Interpolation', (1, 6, 1, 9), ('Name', (1, 7, 1, 8), 'a', ('Load',)), 'a', -1, None), ('Constant', (1, 9, 1, 10), ')', None)])), ] main() diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py index 5309939777f..e2e619c5a23 100644 --- a/Lib/test/test_ast/test_ast.py +++ b/Lib/test/test_ast/test_ast.py @@ -1,34 +1,59 @@ +import _ast_unparse import ast import builtins +import contextlib import copy import dis import enum +import itertools import os import re import sys +import tempfile import textwrap import types import unittest -import warnings import weakref -from functools import partial +from io import StringIO +from pathlib import Path from textwrap import dedent - try: import _testinternalcapi except ImportError: _testinternalcapi = None from test import support -from test.support.import_helper import import_fresh_module -from test.support import os_helper, script_helper +from test.support import os_helper +from test.support import skip_emscripten_stack_overflow, skip_wasi_stack_overflow, skip_if_unlimited_stack_size from test.support.ast_helper import ASTTestMixin +from test.support.import_helper import ensure_lazy_imports from test.test_ast.utils import to_tuple from test.test_ast.snippets import ( eval_tests, eval_results, exec_tests, exec_results, single_tests, single_results ) +STDLIB = os.path.dirname(ast.__file__) +STDLIB_FILES = [fn for fn in os.listdir(STDLIB) if fn.endswith(".py")] +STDLIB_FILES.extend(["test/test_grammar.py", "test/test_unpack_ex.py"]) + +AST_REPR_DATA_FILE = Path(__file__).parent / "data" / "ast_repr.txt" + +def ast_repr_get_test_cases() -> list[str]: + return exec_tests + eval_tests + + +def ast_repr_update_snapshots() -> None: + data = [repr(ast.parse(test)) for test in ast_repr_get_test_cases()] + AST_REPR_DATA_FILE.write_text("\n".join(data)) + + +class LazyImportTest(unittest.TestCase): + @support.cpython_only + def test_lazy_import(self): + ensure_lazy_imports("ast", {"contextlib", "enum", "inspect", "re", "collections", "argparse"}) + + class AST_Tests(unittest.TestCase): maxDiff = None @@ -37,7 +62,7 @@ def _is_ast_node(self, name, node): return False if "ast" not in node.__module__: return False - return name != "AST" and name[0].isupper() + return name != 'AST' and name[0].isupper() def _assertTrueorder(self, ast_node, parent_pos): if not isinstance(ast_node, ast.AST) or ast_node._fields is None: @@ -50,7 +75,7 @@ def _assertTrueorder(self, ast_node, parent_pos): value = getattr(ast_node, name) if isinstance(value, list): first_pos = parent_pos - if value and name == "decorator_list": + if value and name == 'decorator_list': first_pos = (value[0].lineno, value[0].col_offset) for child in value: self._assertTrueorder(child, first_pos) @@ -72,7 +97,6 @@ def test_AST_objects(self): # "ast.AST constructor takes 0 positional arguments" ast.AST(2) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_AST_fields_NULL_check(self): # See: https://github.com/python/cpython/issues/126105 old_value = ast.AST._fields @@ -90,11 +114,10 @@ def cleanup(): with self.assertRaisesRegex(AttributeError, msg): ast.AST() - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: .X object at 0x7e85c3a80> is not None def test_AST_garbage_collection(self): class X: pass - a = ast.AST() a.x = X() a.x.a = a @@ -103,13 +126,10 @@ class X: support.gc_collect() self.assertIsNone(ref()) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_snippets(self): - for input, output, kind in ( - (exec_tests, exec_results, "exec"), - (single_tests, single_results, "single"), - (eval_tests, eval_results, "eval"), - ): + for input, output, kind in ((exec_tests, exec_results, "exec"), + (single_tests, single_results, "single"), + (eval_tests, eval_results, "eval")): for i, o in zip(input, output): with self.subTest(action="parsing", input=i): ast_tree = compile(i, "?", kind, ast.PyCF_ONLY_AST) @@ -118,18 +138,23 @@ def test_snippets(self): with self.subTest(action="compiling", input=i, kind=kind): compile(ast_tree, "?", kind) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_ast_validation(self): # compile() is the only function that calls PyAST_Validate snippets_to_validate = exec_tests + single_tests + eval_tests for snippet in snippets_to_validate: tree = ast.parse(snippet) - compile(tree, "", "exec") + compile(tree, '', 'exec') + + def test_parse_invalid_ast(self): + # see gh-130139 + for optval in (-1, 0, 1, 2): + self.assertRaises(TypeError, ast.parse, ast.Constant(42), + optimize=optval) - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags def test_optimization_levels__debug__(self): - cases = [(-1, "__debug__"), (0, "__debug__"), (1, False), (2, False)] - for optval, expected in cases: + cases = [(-1, '__debug__'), (0, '__debug__'), (1, False), (2, False)] + for (optval, expected) in cases: with self.subTest(optval=optval, expected=expected): res1 = ast.parse("__debug__", optimize=optval) res2 = ast.parse(ast.parse("__debug__"), optimize=optval) @@ -142,33 +167,10 @@ def test_optimization_levels__debug__(self): self.assertIsInstance(res.body[0].value, ast.Name) self.assertEqual(res.body[0].value.id, expected) - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags - def test_optimization_levels_const_folding(self): - folded = ("Expr", (1, 0, 1, 5), ("Constant", (1, 0, 1, 5), 3, None)) - not_folded = ( - "Expr", - (1, 0, 1, 5), - ( - "BinOp", - (1, 0, 1, 5), - ("Constant", (1, 0, 1, 1), 1, None), - ("Add",), - ("Constant", (1, 4, 1, 5), 2, None), - ), - ) - - cases = [(-1, not_folded), (0, not_folded), (1, folded), (2, folded)] - for optval, expected in cases: - with self.subTest(optval=optval): - tree1 = ast.parse("1 + 2", optimize=optval) - tree2 = ast.parse(ast.parse("1 + 2"), optimize=optval) - for tree in [tree1, tree2]: - res = to_tuple(tree.body[0]) - self.assertEqual(res, expected) - - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_invalid_position_information(self): - invalid_linenos = [(10, 1), (-10, -11), (10, -11), (-5, -2), (-5, 1)] + invalid_linenos = [ + (10, 1), (-10, -11), (10, -11), (-5, -2), (-5, 1) + ] for lineno, end_lineno in invalid_linenos: with self.subTest(f"Check invalid linenos {lineno}:{end_lineno}"): @@ -177,36 +179,25 @@ def test_invalid_position_information(self): tree.body[0].lineno = lineno tree.body[0].end_lineno = end_lineno with self.assertRaises(ValueError): - compile(tree, "", "exec") + compile(tree, '', 'exec') - invalid_col_offsets = [(10, 1), (-10, -11), (10, -11), (-5, -2), (-5, 1)] + invalid_col_offsets = [ + (10, 1), (-10, -11), (10, -11), (-5, -2), (-5, 1) + ] for col_offset, end_col_offset in invalid_col_offsets: - with self.subTest( - f"Check invalid col_offset {col_offset}:{end_col_offset}" - ): + with self.subTest(f"Check invalid col_offset {col_offset}:{end_col_offset}"): snippet = "a = 1" tree = ast.parse(snippet) tree.body[0].col_offset = col_offset tree.body[0].end_col_offset = end_col_offset with self.assertRaises(ValueError): - compile(tree, "", "exec") + compile(tree, '', 'exec') def test_compilation_of_ast_nodes_with_default_end_position_values(self): - tree = ast.Module( - body=[ - ast.Import( - names=[ast.alias(name="builtins", lineno=1, col_offset=0)], - lineno=1, - col_offset=0, - ), - ast.Import( - names=[ast.alias(name="traceback", lineno=0, col_offset=0)], - lineno=0, - col_offset=1, - ), - ], - type_ignores=[], - ) + tree = ast.Module(body=[ + ast.Import(names=[ast.alias(name='builtins', lineno=1, col_offset=0)], lineno=1, col_offset=0), + ast.Import(names=[ast.alias(name='traceback', lineno=0, col_offset=0)], lineno=0, col_offset=1) + ], type_ignores=[]) # Check that compilation doesn't crash. Note: this may crash explicitly only on debug mode. compile(tree, "", "exec") @@ -231,6 +222,131 @@ def test_negative_locations_for_compile(self): # This also must not crash: ast.parse(tree, optimize=2) + def test_docstring_optimization_single_node(self): + # https://github.com/python/cpython/issues/137308 + class_example1 = textwrap.dedent(''' + class A: + """Docstring""" + ''') + class_example2 = textwrap.dedent(''' + class A: + """ + Docstring""" + ''') + def_example1 = textwrap.dedent(''' + def some(): + """Docstring""" + ''') + def_example2 = textwrap.dedent(''' + def some(): + """Docstring + """ + ''') + async_def_example1 = textwrap.dedent(''' + async def some(): + """Docstring""" + ''') + async_def_example2 = textwrap.dedent(''' + async def some(): + """ + Docstring + """ + ''') + for code in [ + class_example1, + class_example2, + def_example1, + def_example2, + async_def_example1, + async_def_example2, + ]: + for opt_level in [0, 1, 2]: + with self.subTest(code=code, opt_level=opt_level): + mod = ast.parse(code, optimize=opt_level) + self.assertEqual(len(mod.body[0].body), 1) + if opt_level == 2: + pass_stmt = mod.body[0].body[0] + self.assertIsInstance(pass_stmt, ast.Pass) + self.assertEqual( + vars(pass_stmt), + { + 'lineno': 3, + 'col_offset': 4, + 'end_lineno': 3, + 'end_col_offset': 8, + }, + ) + else: + self.assertIsInstance(mod.body[0].body[0], ast.Expr) + self.assertIsInstance( + mod.body[0].body[0].value, + ast.Constant, + ) + + compile(code, "a", "exec") + compile(code, "a", "exec", optimize=opt_level) + compile(mod, "a", "exec") + compile(mod, "a", "exec", optimize=opt_level) + + def test_docstring_optimization_multiple_nodes(self): + # https://github.com/python/cpython/issues/137308 + class_example = textwrap.dedent( + """ + class A: + ''' + Docstring + ''' + x = 1 + """ + ) + + def_example = textwrap.dedent( + """ + def some(): + ''' + Docstring + + ''' + x = 1 + """ + ) + + async_def_example = textwrap.dedent( + """ + async def some(): + + '''Docstring + + ''' + x = 1 + """ + ) + + for code in [ + class_example, + def_example, + async_def_example, + ]: + for opt_level in [0, 1, 2]: + with self.subTest(code=code, opt_level=opt_level): + mod = ast.parse(code, optimize=opt_level) + if opt_level == 2: + self.assertNotIsInstance( + mod.body[0].body[0], + (ast.Pass, ast.Expr), + ) + else: + self.assertIsInstance(mod.body[0].body[0], ast.Expr) + self.assertIsInstance( + mod.body[0].body[0].value, + ast.Constant, + ) + + compile(code, "a", "exec") + compile(code, "a", "exec", optimize=opt_level) + compile(mod, "a", "exec") + compile(mod, "a", "exec", optimize=opt_level) + def test_slice(self): slc = ast.parse("x[::]").body[0].value.slice self.assertIsNone(slc.upper) @@ -251,7 +367,7 @@ def test_alias(self): im = ast.parse("from bar import y").body[0] self.assertEqual(len(im.names), 1) alias = im.names[0] - self.assertEqual(alias.name, "y") + self.assertEqual(alias.name, 'y') self.assertIsNone(alias.asname) self.assertEqual(alias.lineno, 1) self.assertEqual(alias.end_lineno, 1) @@ -260,7 +376,7 @@ def test_alias(self): im = ast.parse("from bar import *").body[0] alias = im.names[0] - self.assertEqual(alias.name, "*") + self.assertEqual(alias.name, '*') self.assertIsNone(alias.asname) self.assertEqual(alias.lineno, 1) self.assertEqual(alias.end_lineno, 1) @@ -286,45 +402,17 @@ def test_alias(self): self.assertEqual(alias.end_col_offset, 17) def test_base_classes(self): - self.assertTrue(issubclass(ast.For, ast.stmt)) - self.assertTrue(issubclass(ast.Name, ast.expr)) - self.assertTrue(issubclass(ast.stmt, ast.AST)) - self.assertTrue(issubclass(ast.expr, ast.AST)) - self.assertTrue(issubclass(ast.comprehension, ast.AST)) - self.assertTrue(issubclass(ast.Gt, ast.AST)) - - def test_import_deprecated(self): - ast = import_fresh_module("ast") - depr_regex = ( - r"ast\.{} is deprecated and will be removed in Python 3.14; " - r"use ast\.Constant instead" - ) - for name in "Num", "Str", "Bytes", "NameConstant", "Ellipsis": - with self.assertWarnsRegex(DeprecationWarning, depr_regex.format(name)): - getattr(ast, name) - - def test_field_attr_existence_deprecated(self): - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", "", DeprecationWarning) - from ast import Num, Str, Bytes, NameConstant, Ellipsis + self.assertIsSubclass(ast.For, ast.stmt) + self.assertIsSubclass(ast.Name, ast.expr) + self.assertIsSubclass(ast.stmt, ast.AST) + self.assertIsSubclass(ast.expr, ast.AST) + self.assertIsSubclass(ast.comprehension, ast.AST) + self.assertIsSubclass(ast.Gt, ast.AST) - for name in ("Num", "Str", "Bytes", "NameConstant", "Ellipsis"): - item = getattr(ast, name) - if self._is_ast_node(name, item): - with self.subTest(item): - with self.assertWarns(DeprecationWarning): - x = item() - if isinstance(x, ast.AST): - self.assertIs(type(x._fields), tuple) - - @unittest.expectedFailure # TODO: RUSTPYTHON; type object 'Module' has no attribute '__annotations__' def test_field_attr_existence(self): for name, item in ast.__dict__.items(): - # These emit DeprecationWarnings - if name in {"Num", "Str", "Bytes", "NameConstant", "Ellipsis"}: - continue # constructor has a different signature - if name == "Index": + if name == 'Index': continue if self._is_ast_node(name, item): x = self._construct_ast_class(item) @@ -335,42 +423,28 @@ def _construct_ast_class(self, cls): kwargs = {} for name, typ in cls.__annotations__.items(): if typ is str: - kwargs[name] = "capybara" + kwargs[name] = 'capybara' elif typ is int: kwargs[name] = 42 elif typ is object: - kwargs[name] = b"capybara" + kwargs[name] = b'capybara' elif isinstance(typ, type) and issubclass(typ, ast.AST): kwargs[name] = self._construct_ast_class(typ) return cls(**kwargs) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_arguments(self): x = ast.arguments() - self.assertEqual( - x._fields, - ( - "posonlyargs", - "args", - "vararg", - "kwonlyargs", - "kw_defaults", - "kwarg", - "defaults", - ), - ) - self.assertEqual( - x.__annotations__, - { - "posonlyargs": list[ast.arg], - "args": list[ast.arg], - "vararg": ast.arg | None, - "kwonlyargs": list[ast.arg], - "kw_defaults": list[ast.expr], - "kwarg": ast.arg | None, - "defaults": list[ast.expr], - }, - ) + self.assertEqual(x._fields, ('posonlyargs', 'args', 'vararg', 'kwonlyargs', + 'kw_defaults', 'kwarg', 'defaults')) + self.assertEqual(ast.arguments.__annotations__, { + 'posonlyargs': list[ast.arg], + 'args': list[ast.arg], + 'vararg': ast.arg | None, + 'kwonlyargs': list[ast.arg], + 'kw_defaults': list[ast.expr], + 'kwarg': ast.arg | None, + 'defaults': list[ast.expr], + }) self.assertEqual(x.args, []) self.assertIsNone(x.vararg) @@ -379,117 +453,16 @@ def test_arguments(self): self.assertEqual(x.args, 2) self.assertEqual(x.vararg, 3) - def test_field_attr_writable_deprecated(self): - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", "", DeprecationWarning) - x = ast.Num() - # We can assign to _fields - x._fields = 666 - self.assertEqual(x._fields, 666) - def test_field_attr_writable(self): x = ast.Constant(1) # We can assign to _fields x._fields = 666 self.assertEqual(x._fields, 666) - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_classattrs_deprecated(self): - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", "", DeprecationWarning) - from ast import Num, Str, Bytes, NameConstant, Ellipsis - - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings("always", "", DeprecationWarning) - x = ast.Num() - self.assertEqual(x._fields, ("value", "kind")) - - with self.assertRaises(AttributeError): - x.value - - with self.assertRaises(AttributeError): - x.n - - x = ast.Num(42) - self.assertEqual(x.value, 42) - self.assertEqual(x.n, 42) - - with self.assertRaises(AttributeError): - x.lineno - - with self.assertRaises(AttributeError): - x.foobar - - x = ast.Num(lineno=2) - self.assertEqual(x.lineno, 2) - - x = ast.Num(42, lineno=0) - self.assertEqual(x.lineno, 0) - self.assertEqual(x._fields, ("value", "kind")) - self.assertEqual(x.value, 42) - self.assertEqual(x.n, 42) - - self.assertRaises(TypeError, ast.Num, 1, None, 2) - self.assertRaises(TypeError, ast.Num, 1, None, 2, lineno=0) - - # Arbitrary keyword arguments are supported - self.assertEqual(ast.Num(1, foo="bar").foo, "bar") - - with self.assertRaisesRegex( - TypeError, "Num got multiple values for argument 'n'" - ): - ast.Num(1, n=2) - - self.assertEqual(ast.Num(42).n, 42) - self.assertEqual(ast.Num(4.25).n, 4.25) - self.assertEqual(ast.Num(4.25j).n, 4.25j) - self.assertEqual(ast.Str("42").s, "42") - self.assertEqual(ast.Bytes(b"42").s, b"42") - self.assertIs(ast.NameConstant(True).value, True) - self.assertIs(ast.NameConstant(False).value, False) - self.assertIs(ast.NameConstant(None).value, None) - - self.assertEqual( - [str(w.message) for w in wlog], - [ - "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", - "Constant.__init__ missing 1 required positional argument: 'value'. This will become " - "an error in Python 3.15.", - "Attribute n is deprecated and will be removed in Python 3.14; use value instead", - "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", - "Attribute n is deprecated and will be removed in Python 3.14; use value instead", - "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", - "Constant.__init__ missing 1 required positional argument: 'value'. This will become " - "an error in Python 3.15.", - "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", - "Attribute n is deprecated and will be removed in Python 3.14; use value instead", - "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", - "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", - "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", - "Constant.__init__ got an unexpected keyword argument 'foo'. Support for " - "arbitrary keyword arguments is deprecated and will be removed in Python " - "3.15.", - "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", - "Attribute n is deprecated and will be removed in Python 3.14; use value instead", - "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", - "Attribute n is deprecated and will be removed in Python 3.14; use value instead", - "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", - "Attribute n is deprecated and will be removed in Python 3.14; use value instead", - "ast.Str is deprecated and will be removed in Python 3.14; use ast.Constant instead", - "Attribute s is deprecated and will be removed in Python 3.14; use value instead", - "ast.Bytes is deprecated and will be removed in Python 3.14; use ast.Constant instead", - "Attribute s is deprecated and will be removed in Python 3.14; use value instead", - "ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead", - "ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead", - "ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead", - ], - ) - - @unittest.expectedFailure # TODO: RUSTPYTHON; DeprecationWarning not triggered def test_classattrs(self): with self.assertWarns(DeprecationWarning): x = ast.Constant() - self.assertEqual(x._fields, ("value", "kind")) + self.assertEqual(x._fields, ('value', 'kind')) with self.assertRaises(AttributeError): x.value @@ -508,7 +481,7 @@ def test_classattrs(self): x = ast.Constant(42, lineno=0) self.assertEqual(x.lineno, 0) - self.assertEqual(x._fields, ("value", "kind")) + self.assertEqual(x._fields, ('value', 'kind')) self.assertEqual(x.value, 42) self.assertRaises(TypeError, ast.Constant, 1, None, 2) @@ -516,234 +489,32 @@ def test_classattrs(self): # Arbitrary keyword arguments are supported (but deprecated) with self.assertWarns(DeprecationWarning): - self.assertEqual(ast.Constant(1, foo="bar").foo, "bar") + self.assertEqual(ast.Constant(1, foo='bar').foo, 'bar') - with self.assertRaisesRegex( - TypeError, "Constant got multiple values for argument 'value'" - ): + with self.assertRaisesRegex(TypeError, "Constant got multiple values for argument 'value'"): ast.Constant(1, value=2) self.assertEqual(ast.Constant(42).value, 42) self.assertEqual(ast.Constant(4.25).value, 4.25) self.assertEqual(ast.Constant(4.25j).value, 4.25j) - self.assertEqual(ast.Constant("42").value, "42") - self.assertEqual(ast.Constant(b"42").value, b"42") + self.assertEqual(ast.Constant('42').value, '42') + self.assertEqual(ast.Constant(b'42').value, b'42') self.assertIs(ast.Constant(True).value, True) self.assertIs(ast.Constant(False).value, False) self.assertIs(ast.Constant(None).value, None) self.assertIs(ast.Constant(...).value, ...) - def test_realtype(self): - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", "", DeprecationWarning) - from ast import Num, Str, Bytes, NameConstant, Ellipsis - - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings("always", "", DeprecationWarning) - self.assertIs(type(ast.Num(42)), ast.Constant) - self.assertIs(type(ast.Num(4.25)), ast.Constant) - self.assertIs(type(ast.Num(4.25j)), ast.Constant) - self.assertIs(type(ast.Str("42")), ast.Constant) - self.assertIs(type(ast.Bytes(b"42")), ast.Constant) - self.assertIs(type(ast.NameConstant(True)), ast.Constant) - self.assertIs(type(ast.NameConstant(False)), ast.Constant) - self.assertIs(type(ast.NameConstant(None)), ast.Constant) - self.assertIs(type(ast.Ellipsis()), ast.Constant) - - self.assertEqual( - [str(w.message) for w in wlog], - [ - "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", - "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", - "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", - "ast.Str is deprecated and will be removed in Python 3.14; use ast.Constant instead", - "ast.Bytes is deprecated and will be removed in Python 3.14; use ast.Constant instead", - "ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead", - "ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead", - "ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead", - "ast.Ellipsis is deprecated and will be removed in Python 3.14; use ast.Constant instead", - ], - ) - - def test_isinstance(self): - from ast import Constant - - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", "", DeprecationWarning) - from ast import Num, Str, Bytes, NameConstant, Ellipsis - - cls_depr_msg = ( - "ast.{} is deprecated and will be removed in Python 3.14; " - "use ast.Constant instead" - ) - - assertNumDeprecated = partial( - self.assertWarnsRegex, DeprecationWarning, cls_depr_msg.format("Num") - ) - assertStrDeprecated = partial( - self.assertWarnsRegex, DeprecationWarning, cls_depr_msg.format("Str") - ) - assertBytesDeprecated = partial( - self.assertWarnsRegex, DeprecationWarning, cls_depr_msg.format("Bytes") - ) - assertNameConstantDeprecated = partial( - self.assertWarnsRegex, - DeprecationWarning, - cls_depr_msg.format("NameConstant"), - ) - assertEllipsisDeprecated = partial( - self.assertWarnsRegex, DeprecationWarning, cls_depr_msg.format("Ellipsis") - ) - - for arg in 42, 4.2, 4.2j: - with self.subTest(arg=arg): - with assertNumDeprecated(): - n = Num(arg) - with assertNumDeprecated(): - self.assertIsInstance(n, Num) - - with assertStrDeprecated(): - s = Str("42") - with assertStrDeprecated(): - self.assertIsInstance(s, Str) - - with assertBytesDeprecated(): - b = Bytes(b"42") - with assertBytesDeprecated(): - self.assertIsInstance(b, Bytes) - - for arg in True, False, None: - with self.subTest(arg=arg): - with assertNameConstantDeprecated(): - n = NameConstant(arg) - with assertNameConstantDeprecated(): - self.assertIsInstance(n, NameConstant) - - with assertEllipsisDeprecated(): - e = Ellipsis() - with assertEllipsisDeprecated(): - self.assertIsInstance(e, Ellipsis) - - for arg in 42, 4.2, 4.2j: - with self.subTest(arg=arg): - with assertNumDeprecated(): - self.assertIsInstance(Constant(arg), Num) - - with assertStrDeprecated(): - self.assertIsInstance(Constant("42"), Str) - - with assertBytesDeprecated(): - self.assertIsInstance(Constant(b"42"), Bytes) - - for arg in True, False, None: - with self.subTest(arg=arg): - with assertNameConstantDeprecated(): - self.assertIsInstance(Constant(arg), NameConstant) - - with assertEllipsisDeprecated(): - self.assertIsInstance(Constant(...), Ellipsis) - - with assertStrDeprecated(): - s = Str("42") - assertNumDeprecated(self.assertNotIsInstance, s, Num) - assertBytesDeprecated(self.assertNotIsInstance, s, Bytes) - - with assertNumDeprecated(): - n = Num(42) - assertStrDeprecated(self.assertNotIsInstance, n, Str) - assertNameConstantDeprecated(self.assertNotIsInstance, n, NameConstant) - assertEllipsisDeprecated(self.assertNotIsInstance, n, Ellipsis) - - with assertNameConstantDeprecated(): - n = NameConstant(True) - with assertNumDeprecated(): - self.assertNotIsInstance(n, Num) - - with assertNameConstantDeprecated(): - n = NameConstant(False) - with assertNumDeprecated(): - self.assertNotIsInstance(n, Num) - - for arg in "42", True, False: - with self.subTest(arg=arg): - with assertNumDeprecated(): - self.assertNotIsInstance(Constant(arg), Num) - - assertStrDeprecated(self.assertNotIsInstance, Constant(42), Str) - assertBytesDeprecated(self.assertNotIsInstance, Constant("42"), Bytes) - assertNameConstantDeprecated( - self.assertNotIsInstance, Constant(42), NameConstant - ) - assertEllipsisDeprecated(self.assertNotIsInstance, Constant(42), Ellipsis) - assertNumDeprecated(self.assertNotIsInstance, Constant(None), Num) - assertStrDeprecated(self.assertNotIsInstance, Constant(None), Str) - assertBytesDeprecated(self.assertNotIsInstance, Constant(None), Bytes) - assertNameConstantDeprecated( - self.assertNotIsInstance, Constant(1), NameConstant - ) - assertEllipsisDeprecated(self.assertNotIsInstance, Constant(None), Ellipsis) - - class S(str): - pass - - with assertStrDeprecated(): - self.assertIsInstance(Constant(S("42")), Str) - with assertNumDeprecated(): - self.assertNotIsInstance(Constant(S("42")), Num) - - @unittest.expectedFailure # TODO: RUSTPYTHON; will be removed in Python 3.14 - def test_constant_subclasses_deprecated(self): - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", "", DeprecationWarning) - from ast import Num - - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings("always", "", DeprecationWarning) - - class N(ast.Num): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.z = "spam" - - class N2(ast.Num): - pass - - n = N(42) - self.assertEqual(n.n, 42) - self.assertEqual(n.z, "spam") - self.assertIs(type(n), N) - self.assertIsInstance(n, N) - self.assertIsInstance(n, ast.Num) - self.assertNotIsInstance(n, N2) - self.assertNotIsInstance(ast.Num(42), N) - n = N(n=42) - self.assertEqual(n.n, 42) - self.assertIs(type(n), N) - - self.assertEqual( - [str(w.message) for w in wlog], - [ - "Attribute n is deprecated and will be removed in Python 3.14; use value instead", - "Attribute n is deprecated and will be removed in Python 3.14; use value instead", - "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", - "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", - "Attribute n is deprecated and will be removed in Python 3.14; use value instead", - "Attribute n is deprecated and will be removed in Python 3.14; use value instead", - ], - ) - def test_constant_subclasses(self): class N(ast.Constant): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.z = "spam" - + self.z = 'spam' class N2(ast.Constant): pass n = N(42) self.assertEqual(n.value, 42) - self.assertEqual(n.z, "spam") + self.assertEqual(n.z, 'spam') self.assertEqual(type(n), N) self.assertTrue(isinstance(n, N)) self.assertTrue(isinstance(n, ast.Constant)) @@ -758,12 +529,11 @@ def test_module(self): x = ast.Module(body, []) self.assertEqual(x.body, body) - @unittest.expectedFailure # TODO: RUSTPYTHON; DeprecationWarning not triggered def test_nodeclasses(self): # Zero arguments constructor explicitly allowed (but deprecated) with self.assertWarns(DeprecationWarning): x = ast.BinOp() - self.assertEqual(x._fields, ("left", "op", "right")) + self.assertEqual(x._fields, ('left', 'op', 'right')) # Random attribute allowed too x.foobarbaz = 5 @@ -810,15 +580,14 @@ def test_no_fields(self): x = ast.Sub() self.assertEqual(x._fields, ()) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_invalid_sum(self): pos = dict(lineno=2, col_offset=3) m = ast.Module([ast.Expr(ast.expr(**pos), **pos)], []) with self.assertRaises(TypeError) as cm: compile(m, "", "exec") - self.assertIn("but got ", "eval") - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: expected some sort of expr, but got None def test_empty_yield_from(self): # Issue 16546: yield from value is not optional. empty_yield_from = ast.parse("def f():\n yield from g()") @@ -847,15 +618,13 @@ def test_issue31592(self): # There shouldn't be an assertion failure in case of a bad # unicodedata.normalize(). import unicodedata - def bad_normalize(*args): return None - - with support.swap_attr(unicodedata, "normalize", bad_normalize): - self.assertRaises(TypeError, ast.parse, "\u03d5") + with support.swap_attr(unicodedata, 'normalize', bad_normalize): + self.assertRaises(TypeError, ast.parse, '\u03D5') def test_issue18374_binop_col_offset(self): - tree = ast.parse("4+5+6+7") + tree = ast.parse('4+5+6+7') parent_binop = tree.body[0].value child_binop = parent_binop.left grandchild_binop = child_binop.left @@ -866,7 +635,7 @@ def test_issue18374_binop_col_offset(self): self.assertEqual(grandchild_binop.col_offset, 0) self.assertEqual(grandchild_binop.end_col_offset, 3) - tree = ast.parse("4+5-\\\n 6-7") + tree = ast.parse('4+5-\\\n 6-7') parent_binop = tree.body[0].value child_binop = parent_binop.left grandchild_binop = child_binop.left @@ -886,62 +655,264 @@ def test_issue18374_binop_col_offset(self): self.assertEqual(grandchild_binop.end_lineno, 1) def test_issue39579_dotted_name_end_col_offset(self): - tree = ast.parse("@a.b.c\ndef f(): pass") + tree = ast.parse('@a.b.c\ndef f(): pass') attr_b = tree.body[0].decorator_list[0].value self.assertEqual(attr_b.end_col_offset, 4) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_ast_asdl_signature(self): - self.assertEqual( - ast.withitem.__doc__, "withitem(expr context_expr, expr? optional_vars)" - ) + self.assertEqual(ast.withitem.__doc__, "withitem(expr context_expr, expr? optional_vars)") self.assertEqual(ast.GtE.__doc__, "GtE") self.assertEqual(ast.Name.__doc__, "Name(identifier id, expr_context ctx)") - self.assertEqual( - ast.cmpop.__doc__, - "cmpop = Eq | NotEq | Lt | LtE | Gt | GtE | Is | IsNot | In | NotIn", - ) + self.assertEqual(ast.cmpop.__doc__, "cmpop = Eq | NotEq | Lt | LtE | Gt | GtE | Is | IsNot | In | NotIn") expressions = [f" | {node.__doc__}" for node in ast.expr.__subclasses__()] expressions[0] = f"expr = {ast.expr.__subclasses__()[0].__doc__}" self.assertCountEqual(ast.expr.__doc__.split("\n"), expressions) - @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError not raised + def test_compare_basics(self): + self.assertTrue(ast.compare(ast.parse("x = 10"), ast.parse("x = 10"))) + self.assertFalse(ast.compare(ast.parse("x = 10"), ast.parse(""))) + self.assertFalse(ast.compare(ast.parse("x = 10"), ast.parse("x"))) + self.assertFalse( + ast.compare(ast.parse("x = 10;y = 20"), ast.parse("class C:pass")) + ) + + def test_compare_modified_ast(self): + # The ast API is a bit underspecified. The objects are mutable, + # and even _fields and _attributes are mutable. The compare() does + # some simple things to accommodate mutability. + a = ast.parse("m * x + b", mode="eval") + b = ast.parse("m * x + b", mode="eval") + self.assertTrue(ast.compare(a, b)) + + a._fields = a._fields + ("spam",) + a.spam = "Spam" + self.assertNotEqual(a._fields, b._fields) + self.assertFalse(ast.compare(a, b)) + self.assertFalse(ast.compare(b, a)) + + b._fields = a._fields + b.spam = a.spam + self.assertTrue(ast.compare(a, b)) + self.assertTrue(ast.compare(b, a)) + + b._attributes = b._attributes + ("eggs",) + b.eggs = "eggs" + self.assertNotEqual(a._attributes, b._attributes) + self.assertFalse(ast.compare(a, b, compare_attributes=True)) + self.assertFalse(ast.compare(b, a, compare_attributes=True)) + + a._attributes = b._attributes + a.eggs = b.eggs + self.assertTrue(ast.compare(a, b, compare_attributes=True)) + self.assertTrue(ast.compare(b, a, compare_attributes=True)) + + def test_compare_literals(self): + constants = ( + -20, + 20, + 20.0, + 1, + 1.0, + True, + 0, + False, + frozenset(), + tuple(), + "ABCD", + "abcd", + "中文字", + 1e1000, + -1e1000, + ) + for next_index, constant in enumerate(constants[:-1], 1): + next_constant = constants[next_index] + with self.subTest(literal=constant, next_literal=next_constant): + self.assertTrue( + ast.compare(ast.Constant(constant), ast.Constant(constant)) + ) + self.assertFalse( + ast.compare( + ast.Constant(constant), ast.Constant(next_constant) + ) + ) + + same_looking_literal_cases = [ + {1, 1.0, True, 1 + 0j}, + {0, 0.0, False, 0 + 0j}, + ] + for same_looking_literals in same_looking_literal_cases: + for literal in same_looking_literals: + for same_looking_literal in same_looking_literals - {literal}: + self.assertFalse( + ast.compare( + ast.Constant(literal), + ast.Constant(same_looking_literal), + ) + ) + + def test_compare_fieldless(self): + self.assertTrue(ast.compare(ast.Add(), ast.Add())) + self.assertFalse(ast.compare(ast.Sub(), ast.Add())) + + # test that missing runtime fields is handled in ast.compare() + a1, a2 = ast.Name('a'), ast.Name('a') + self.assertTrue(ast.compare(a1, a2)) + self.assertTrue(ast.compare(a1, a2)) + del a1.id + self.assertFalse(ast.compare(a1, a2)) + del a2.id + self.assertTrue(ast.compare(a1, a2)) + + def test_compare_modes(self): + for mode, sources in ( + ("exec", exec_tests), + ("eval", eval_tests), + ("single", single_tests), + ): + for source in sources: + a = ast.parse(source, mode=mode) + b = ast.parse(source, mode=mode) + self.assertTrue( + ast.compare(a, b), f"{ast.dump(a)} != {ast.dump(b)}" + ) + + def test_compare_attributes_option(self): + def parse(a, b): + return ast.parse(a), ast.parse(b) + + a, b = parse("2 + 2", "2+2") + self.assertTrue(ast.compare(a, b)) + self.assertTrue(ast.compare(a, b, compare_attributes=False)) + self.assertFalse(ast.compare(a, b, compare_attributes=True)) + + def test_compare_attributes_option_missing_attribute(self): + # test that missing runtime attributes is handled in ast.compare() + a1, a2 = ast.Name('a', lineno=1), ast.Name('a', lineno=1) + self.assertTrue(ast.compare(a1, a2)) + self.assertTrue(ast.compare(a1, a2, compare_attributes=True)) + del a1.lineno + self.assertFalse(ast.compare(a1, a2, compare_attributes=True)) + del a2.lineno + self.assertTrue(ast.compare(a1, a2, compare_attributes=True)) + def test_positional_only_feature_version(self): - ast.parse("def foo(x, /): ...", feature_version=(3, 8)) - ast.parse("def bar(x=1, /): ...", feature_version=(3, 8)) + ast.parse('def foo(x, /): ...', feature_version=(3, 8)) + ast.parse('def bar(x=1, /): ...', feature_version=(3, 8)) with self.assertRaises(SyntaxError): - ast.parse("def foo(x, /): ...", feature_version=(3, 7)) + ast.parse('def foo(x, /): ...', feature_version=(3, 7)) with self.assertRaises(SyntaxError): - ast.parse("def bar(x=1, /): ...", feature_version=(3, 7)) + ast.parse('def bar(x=1, /): ...', feature_version=(3, 7)) - ast.parse("lambda x, /: ...", feature_version=(3, 8)) - ast.parse("lambda x=1, /: ...", feature_version=(3, 8)) + ast.parse('lambda x, /: ...', feature_version=(3, 8)) + ast.parse('lambda x=1, /: ...', feature_version=(3, 8)) with self.assertRaises(SyntaxError): - ast.parse("lambda x, /: ...", feature_version=(3, 7)) + ast.parse('lambda x, /: ...', feature_version=(3, 7)) with self.assertRaises(SyntaxError): - ast.parse("lambda x=1, /: ...", feature_version=(3, 7)) + ast.parse('lambda x=1, /: ...', feature_version=(3, 7)) - @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError not raised def test_assignment_expression_feature_version(self): - ast.parse("(x := 0)", feature_version=(3, 8)) + ast.parse('(x := 0)', feature_version=(3, 8)) + with self.assertRaises(SyntaxError): + ast.parse('(x := 0)', feature_version=(3, 7)) + + def test_pep750_tstring(self): + code = 't""' + ast.parse(code, feature_version=(3, 14)) + with self.assertRaises(SyntaxError): + ast.parse(code, feature_version=(3, 13)) + + def test_pep758_except_without_parens(self): + code = textwrap.dedent(""" + try: + ... + except ValueError, TypeError: + ... + """) + ast.parse(code, feature_version=(3, 14)) with self.assertRaises(SyntaxError): - ast.parse("(x := 0)", feature_version=(3, 7)) + ast.parse(code, feature_version=(3, 13)) + + def test_pep758_except_with_single_expr(self): + single_expr = textwrap.dedent(""" + try: + ... + except{0} TypeError: + ... + """) + + single_expr_with_as = textwrap.dedent(""" + try: + ... + except{0} TypeError as exc: + ... + """) + + single_tuple_expr = textwrap.dedent(""" + try: + ... + except{0} (TypeError,): + ... + """) + + single_tuple_expr_with_as = textwrap.dedent(""" + try: + ... + except{0} (TypeError,) as exc: + ... + """) + + single_parens_expr = textwrap.dedent(""" + try: + ... + except{0} (TypeError): + ... + """) + + single_parens_expr_with_as = textwrap.dedent(""" + try: + ... + except{0} (TypeError) as exc: + ... + """) + + for code in [ + single_expr, + single_expr_with_as, + single_tuple_expr, + single_tuple_expr_with_as, + single_parens_expr, + single_parens_expr_with_as, + ]: + for star in [True, False]: + code = code.format('*' if star else '') + with self.subTest(code=code, star=star): + ast.parse(code, feature_version=(3, 14)) + ast.parse(code, feature_version=(3, 13)) + + def test_pep758_except_star_without_parens(self): + code = textwrap.dedent(""" + try: + ... + except* ValueError, TypeError: + ... + """) + ast.parse(code, feature_version=(3, 14)) + with self.assertRaises(SyntaxError): + ast.parse(code, feature_version=(3, 13)) def test_conditional_context_managers_parse_with_low_feature_version(self): # regression test for gh-115881 - ast.parse("with (x() if y else z()): ...", feature_version=(3, 8)) + ast.parse('with (x() if y else z()): ...', feature_version=(3, 8)) - @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError not raised def test_exception_groups_feature_version(self): - code = dedent(""" + code = dedent(''' try: ... except* Exception: ... - """) + ''') ast.parse(code) with self.assertRaises(SyntaxError): ast.parse(code, feature_version=(3, 10)) - @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError not raised def test_type_params_feature_version(self): samples = [ "type X = int", @@ -954,7 +925,7 @@ def test_type_params_feature_version(self): with self.assertRaises(SyntaxError): ast.parse(sample, feature_version=(3, 11)) - @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError not raised + @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError not raised def test_type_params_default_feature_version(self): samples = [ "type X[*Ts=int] = int", @@ -969,21 +940,18 @@ def test_type_params_default_feature_version(self): def test_invalid_major_feature_version(self): with self.assertRaises(ValueError): - ast.parse("pass", feature_version=(2, 7)) + ast.parse('pass', feature_version=(2, 7)) with self.assertRaises(ValueError): - ast.parse("pass", feature_version=(4, 0)) + ast.parse('pass', feature_version=(4, 0)) - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_constant_as_name(self): for constant in "True", "False", "None": expr = ast.Expression(ast.Name(constant, ast.Load())) ast.fix_missing_locations(expr) - with self.assertRaisesRegex( - ValueError, f"identifier field can't represent '{constant}' constant" - ): + with self.assertRaisesRegex(ValueError, f"identifier field can't represent '{constant}' constant"): compile(expr, "", "eval") - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_constant_as_unicode_name(self): constants = [ ("True", b"Tru\xe1\xb5\x89"), @@ -998,41 +966,41 @@ def test_constant_as_unicode_name(self): def test_precedence_enum(self): class _Precedence(enum.IntEnum): """Precedence table that originated from python grammar.""" - - NAMED_EXPR = enum.auto() # := - TUPLE = enum.auto() # , - YIELD = enum.auto() # 'yield', 'yield from' - TEST = enum.auto() # 'if'-'else', 'lambda' - OR = enum.auto() # 'or' - AND = enum.auto() # 'and' - NOT = enum.auto() # 'not' - CMP = enum.auto() # '<', '>', '==', '>=', '<=', '!=', - # 'in', 'not in', 'is', 'is not' + NAMED_EXPR = enum.auto() # := + TUPLE = enum.auto() # , + YIELD = enum.auto() # 'yield', 'yield from' + TEST = enum.auto() # 'if'-'else', 'lambda' + OR = enum.auto() # 'or' + AND = enum.auto() # 'and' + NOT = enum.auto() # 'not' + CMP = enum.auto() # '<', '>', '==', '>=', '<=', '!=', + # 'in', 'not in', 'is', 'is not' EXPR = enum.auto() - BOR = EXPR # '|' - BXOR = enum.auto() # '^' - BAND = enum.auto() # '&' - SHIFT = enum.auto() # '<<', '>>' - ARITH = enum.auto() # '+', '-' - TERM = enum.auto() # '*', '@', '/', '%', '//' - FACTOR = enum.auto() # unary '+', '-', '~' - POWER = enum.auto() # '**' - AWAIT = enum.auto() # 'await' + BOR = EXPR # '|' + BXOR = enum.auto() # '^' + BAND = enum.auto() # '&' + SHIFT = enum.auto() # '<<', '>>' + ARITH = enum.auto() # '+', '-' + TERM = enum.auto() # '*', '@', '/', '%', '//' + FACTOR = enum.auto() # unary '+', '-', '~' + POWER = enum.auto() # '**' + AWAIT = enum.auto() # 'await' ATOM = enum.auto() - def next(self): try: return self.__class__(self + 1) except ValueError: return self - - enum._test_simple_enum(_Precedence, ast._Precedence) + enum._test_simple_enum(_Precedence, _ast_unparse._Precedence) @support.cpython_only + @skip_if_unlimited_stack_size + @skip_wasi_stack_overflow() + @skip_emscripten_stack_overflow() def test_ast_recursion_limit(self): - fail_depth = support.exceeds_recursion_limit() - crash_depth = 100_000 - success_depth = int(support.get_c_recursion_limit() * 0.8) + # Android test devices have less memory. + crash_depth = 100_000 if sys.platform == "android" else 500_000 + success_depth = 200 if _testinternalcapi is not None: remaining = _testinternalcapi.get_c_recursion_remaining() success_depth = min(success_depth, remaining) @@ -1040,12 +1008,13 @@ def test_ast_recursion_limit(self): def check_limit(prefix, repeated): expect_ok = prefix + repeated * success_depth ast.parse(expect_ok) - for depth in (fail_depth, crash_depth): - broken = prefix + repeated * depth - details = "Compiling ({!r} + {!r} * {})".format(prefix, repeated, depth) - with self.assertRaises(RecursionError, msg=details): - with support.infinite_recursion(): - ast.parse(broken) + + broken = prefix + repeated * crash_depth + details = "Compiling ({!r} + {!r} * {})".format( + prefix, repeated, crash_depth) + with self.assertRaises(RecursionError, msg=details): + with support.infinite_recursion(): + ast.parse(broken) check_limit("a", "()") check_limit("a", ".b") @@ -1053,9 +1022,8 @@ def check_limit(prefix, repeated): check_limit("a", "*a") def test_null_bytes(self): - with self.assertRaises( - SyntaxError, msg="source code string cannot contain null bytes" - ): + with self.assertRaises(SyntaxError, + msg="source code string cannot contain null bytes"): ast.parse("a\0b") def assert_none_check(self, node: type[ast.AST], attr: str, source: str) -> None: @@ -1071,7 +1039,7 @@ def assert_none_check(self, node: type[ast.AST], attr: str, source: str) -> None with self.assertRaisesRegex(ValueError, f"^{e}$"): compile(tree, "", "exec") - @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: expected some sort of expr, but got None + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: expected some sort of expr, but got None def test_none_checks(self) -> None: tests = [ (ast.alias, "name", "import spam as SPAM"), @@ -1085,11 +1053,63 @@ def test_none_checks(self) -> None: for node, attr, source in tests: self.assert_none_check(node, attr, source) + def test_repr(self) -> None: + snapshots = AST_REPR_DATA_FILE.read_text().split("\n") + for test, snapshot in zip(ast_repr_get_test_cases(), snapshots, strict=True): + with self.subTest(test_input=test): + self.assertEqual(repr(ast.parse(test)), snapshot) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: ValueError not raised + def test_repr_large_input_crash(self): + # gh-125010: Fix use-after-free in ast repr() + source = "0x0" + "e" * 10_000 + with self.assertRaisesRegex(ValueError, + r"Exceeds the limit \(\d+ digits\)"): + repr(ast.Constant(value=eval(source))) + + def test_tstring(self): + # Test AST structure for simple t-string + tree = ast.parse('t"Hello"') + self.assertIsInstance(tree.body[0].value, ast.TemplateStr) + self.assertIsInstance(tree.body[0].value.values[0], ast.Constant) + + # Test AST for t-string with interpolation + tree = ast.parse('t"Hello {name}"') + self.assertIsInstance(tree.body[0].value, ast.TemplateStr) + self.assertIsInstance(tree.body[0].value.values[0], ast.Constant) + self.assertIsInstance(tree.body[0].value.values[1], ast.Interpolation) + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags + def test_optimization_levels_const_folding(self): + return super().test_optimization_levels_const_folding() + + @unittest.expectedFailure # TODO: RUSTPYTHON; will be removed in Python 3.14 + def test_constant_subclasses_deprecated(self): + return super().test_constant_subclasses_deprecated() + class CopyTests(unittest.TestCase): """Test copying and pickling AST nodes.""" - @unittest.expectedFailure # TODO: RUSTPYTHON + @staticmethod + def iter_ast_classes(): + """Iterate over the (native) subclasses of ast.AST recursively. + + This excludes the special class ast.Index since its constructor + returns an integer. + """ + def do(cls): + if cls.__module__ != 'ast': + return + if cls is ast.Index: + return + + yield cls + for sub in cls.__subclasses__(): + yield from do(sub) + + yield from do(ast.AST) + def test_pickling(self): import pickle @@ -1100,6 +1120,7 @@ def test_pickling(self): ast2 = pickle.loads(pickle.dumps(tree, protocol)) self.assertEqual(to_tuple(ast2), to_tuple(tree)) + @skip_if_unlimited_stack_size def test_copy_with_parents(self): # gh-120108 code = """ @@ -1154,66 +1175,317 @@ def test_copy_with_parents(self): for node in ast.walk(tree2): for child in ast.iter_child_nodes(node): - if hasattr(child, "parent") and not isinstance( - child, - ( - ast.expr_context, - ast.boolop, - ast.unaryop, - ast.cmpop, - ast.operator, - ), - ): + if hasattr(child, "parent") and not isinstance(child, ( + ast.expr_context, ast.boolop, ast.unaryop, ast.cmpop, ast.operator, + )): self.assertEqual(to_tuple(child.parent), to_tuple(node)) + def test_replace_interface(self): + for klass in self.iter_ast_classes(): + with self.subTest(klass=klass): + self.assertHasAttr(klass, '__replace__') + + fields = set(klass._fields) + with self.subTest(klass=klass, fields=fields): + node = klass(**dict.fromkeys(fields)) + # forbid positional arguments in replace() + self.assertRaises(TypeError, copy.replace, node, 1) + self.assertRaises(TypeError, node.__replace__, 1) + + def test_replace_native(self): + for klass in self.iter_ast_classes(): + fields = set(klass._fields) + attributes = set(klass._attributes) + + with self.subTest(klass=klass, fields=fields, attributes=attributes): + # use of object() to ensure that '==' and 'is' + # behave similarly in ast.compare(node, repl) + old_fields = {field: object() for field in fields} + old_attrs = {attr: object() for attr in attributes} + + # check shallow copy + node = klass(**old_fields) + repl = copy.replace(node) + self.assertTrue(ast.compare(node, repl, compare_attributes=True)) + # check when passing using attributes (they may be optional!) + node = klass(**old_fields, **old_attrs) + repl = copy.replace(node) + self.assertTrue(ast.compare(node, repl, compare_attributes=True)) + + for field in fields: + # check when we sometimes have attributes and sometimes not + for init_attrs in [{}, old_attrs]: + node = klass(**old_fields, **init_attrs) + # only change a single field (do not change attributes) + new_value = object() + repl = copy.replace(node, **{field: new_value}) + for f in fields: + old_value = old_fields[f] + # assert that there is no side-effect + self.assertIs(getattr(node, f), old_value) + # check the changes + if f != field: + self.assertIs(getattr(repl, f), old_value) + else: + self.assertIs(getattr(repl, f), new_value) + self.assertFalse(ast.compare(node, repl, compare_attributes=True)) + + for attribute in attributes: + node = klass(**old_fields, **old_attrs) + # only change a single attribute (do not change fields) + new_attr = object() + repl = copy.replace(node, **{attribute: new_attr}) + for a in attributes: + old_attr = old_attrs[a] + # assert that there is no side-effect + self.assertIs(getattr(node, a), old_attr) + # check the changes + if a != attribute: + self.assertIs(getattr(repl, a), old_attr) + else: + self.assertIs(getattr(repl, a), new_attr) + self.assertFalse(ast.compare(node, repl, compare_attributes=True)) + + def test_replace_accept_known_class_fields(self): + nid, ctx = object(), object() + + node = ast.Name(id=nid, ctx=ctx) + self.assertIs(node.id, nid) + self.assertIs(node.ctx, ctx) + + new_nid = object() + repl = copy.replace(node, id=new_nid) + # assert that there is no side-effect + self.assertIs(node.id, nid) + self.assertIs(node.ctx, ctx) + # check the changes + self.assertIs(repl.id, new_nid) + self.assertIs(repl.ctx, node.ctx) # no changes + + def test_replace_accept_known_class_attributes(self): + node = ast.parse('x').body[0].value + self.assertEqual(node.id, 'x') + self.assertEqual(node.lineno, 1) + + # constructor allows any type so replace() should do the same + lineno = object() + repl = copy.replace(node, lineno=lineno) + # assert that there is no side-effect + self.assertEqual(node.lineno, 1) + # check the changes + self.assertEqual(repl.id, node.id) + self.assertEqual(repl.ctx, node.ctx) + self.assertEqual(repl.lineno, lineno) + + _, _, state = node.__reduce__() + self.assertEqual(state['id'], 'x') + self.assertEqual(state['ctx'], node.ctx) + self.assertEqual(state['lineno'], 1) + + _, _, state = repl.__reduce__() + self.assertEqual(state['id'], 'x') + self.assertEqual(state['ctx'], node.ctx) + self.assertEqual(state['lineno'], lineno) + + def test_replace_accept_known_custom_class_fields(self): + class MyNode(ast.AST): + _fields = ('name', 'data') + __annotations__ = {'name': str, 'data': object} + __match_args__ = ('name', 'data') + + name, data = 'name', object() + + node = MyNode(name, data) + self.assertIs(node.name, name) + self.assertIs(node.data, data) + # check shallow copy + repl = copy.replace(node) + # assert that there is no side-effect + self.assertIs(node.name, name) + self.assertIs(node.data, data) + # check the shallow copy + self.assertIs(repl.name, name) + self.assertIs(repl.data, data) + + node = MyNode(name, data) + repl_data = object() + # replace custom but known field + repl = copy.replace(node, data=repl_data) + # assert that there is no side-effect + self.assertIs(node.name, name) + self.assertIs(node.data, data) + # check the changes + self.assertIs(repl.name, node.name) + self.assertIs(repl.data, repl_data) + + def test_replace_accept_known_custom_class_attributes(self): + class MyNode(ast.AST): + x = 0 + y = 1 + _attributes = ('x', 'y') + + node = MyNode() + self.assertEqual(node.x, 0) + self.assertEqual(node.y, 1) + + y = object() + repl = copy.replace(node, y=y) + # assert that there is no side-effect + self.assertEqual(node.x, 0) + self.assertEqual(node.y, 1) + # check the changes + self.assertEqual(repl.x, 0) + self.assertEqual(repl.y, y) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 'x' is not 'x' + def test_replace_ignore_known_custom_instance_fields(self): + node = ast.parse('x').body[0].value + node.extra = extra = object() # add instance 'extra' field + context = node.ctx + + # assert initial values + self.assertIs(node.id, 'x') + self.assertIs(node.ctx, context) + self.assertIs(node.extra, extra) + # shallow copy, but drops extra fields + repl = copy.replace(node) + # assert that there is no side-effect + self.assertIs(node.id, 'x') + self.assertIs(node.ctx, context) + self.assertIs(node.extra, extra) + # verify that the 'extra' field is not kept + self.assertIs(repl.id, 'x') + self.assertIs(repl.ctx, context) + self.assertRaises(AttributeError, getattr, repl, 'extra') + + # change known native field + repl = copy.replace(node, id='y') + # assert that there is no side-effect + self.assertIs(node.id, 'x') + self.assertIs(node.ctx, context) + self.assertIs(node.extra, extra) + # verify that the 'extra' field is not kept + self.assertIs(repl.id, 'y') + self.assertIs(repl.ctx, context) + self.assertRaises(AttributeError, getattr, repl, 'extra') + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: "Name\.__replace__\ missing\ 1\ keyword\ argument:\ 'id'\." does not match "replace() does not support Name objects" + def test_replace_reject_missing_field(self): + # case: warn if deleted field is not replaced + node = ast.parse('x').body[0].value + context = node.ctx + del node.id + + self.assertRaises(AttributeError, getattr, node, 'id') + self.assertIs(node.ctx, context) + msg = "Name.__replace__ missing 1 keyword argument: 'id'." + with self.assertRaisesRegex(TypeError, re.escape(msg)): + copy.replace(node) + # assert that there is no side-effect + self.assertRaises(AttributeError, getattr, node, 'id') + self.assertIs(node.ctx, context) + + # case: do not raise if deleted field is replaced + node = ast.parse('x').body[0].value + context = node.ctx + del node.id + + self.assertRaises(AttributeError, getattr, node, 'id') + self.assertIs(node.ctx, context) + repl = copy.replace(node, id='y') + # assert that there is no side-effect + self.assertRaises(AttributeError, getattr, node, 'id') + self.assertIs(node.ctx, context) + self.assertIs(repl.id, 'y') + self.assertIs(repl.ctx, context) + + def test_replace_accept_missing_field_with_default(self): + node = ast.FunctionDef(name="foo", args=ast.arguments()) + self.assertIs(node.returns, None) + self.assertEqual(node.decorator_list, []) + node2 = copy.replace(node, name="bar") + self.assertEqual(node2.name, "bar") + self.assertIs(node2.returns, None) + self.assertEqual(node2.decorator_list, []) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: "Name\.__replace__\ got\ an\ unexpected\ keyword\ argument\ 'extra'\." does not match "replace() does not support Name objects" + def test_replace_reject_known_custom_instance_fields_commits(self): + node = ast.parse('x').body[0].value + node.extra = extra = object() # add instance 'extra' field + context = node.ctx + + # explicit rejection of known instance fields + self.assertHasAttr(node, 'extra') + msg = "Name.__replace__ got an unexpected keyword argument 'extra'." + with self.assertRaisesRegex(TypeError, re.escape(msg)): + copy.replace(node, extra=1) + # assert that there is no side-effect + self.assertIs(node.id, 'x') + self.assertIs(node.ctx, context) + self.assertIs(node.extra, extra) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: "Name\.__replace__\ got\ an\ unexpected\ keyword\ argument\ 'unknown'\." does not match "replace() does not support Name objects" + def test_replace_reject_unknown_instance_fields(self): + node = ast.parse('x').body[0].value + context = node.ctx + + # explicit rejection of unknown extra fields + self.assertRaises(AttributeError, getattr, node, 'unknown') + msg = "Name.__replace__ got an unexpected keyword argument 'unknown'." + with self.assertRaisesRegex(TypeError, re.escape(msg)): + copy.replace(node, unknown=1) + # assert that there is no side-effect + self.assertIs(node.id, 'x') + self.assertIs(node.ctx, context) + self.assertRaises(AttributeError, getattr, node, 'unknown') + + @unittest.expectedFailure # TODO: RUSTPYTHON; Wrong error message + def test_replace_non_str_kwarg(self): + node = ast.Name(id="x") + errmsg = "got an unexpected keyword argument ", "exec", ast.PyCF_ONLY_AST) + a = ast.parse('foo(1 + 1)') + b = compile('foo(1 + 1)', '', 'exec', ast.PyCF_ONLY_AST) self.assertEqual(ast.dump(a), ast.dump(b)) def test_parse_in_error(self): try: - 1 / 0 + 1/0 except Exception: with self.assertRaises(SyntaxError) as e: ast.literal_eval(r"'\U'") self.assertIsNotNone(e.exception.__context__) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_dump(self): node = ast.parse('spam(eggs, "and cheese")') - self.assertEqual( - ast.dump(node), + self.assertEqual(ast.dump(node), "Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), " - "args=[Name(id='eggs', ctx=Load()), Constant(value='and cheese')]))])", + "args=[Name(id='eggs', ctx=Load()), Constant(value='and cheese')]))])" ) - self.assertEqual( - ast.dump(node, annotate_fields=False), + self.assertEqual(ast.dump(node, annotate_fields=False), "Module([Expr(Call(Name('spam', Load()), [Name('eggs', Load()), " - "Constant('and cheese')]))])", + "Constant('and cheese')]))])" ) - self.assertEqual( - ast.dump(node, include_attributes=True), + self.assertEqual(ast.dump(node, include_attributes=True), "Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load(), " "lineno=1, col_offset=0, end_lineno=1, end_col_offset=4), " "args=[Name(id='eggs', ctx=Load(), lineno=1, col_offset=5, " "end_lineno=1, end_col_offset=9), Constant(value='and cheese', " "lineno=1, col_offset=11, end_lineno=1, end_col_offset=23)], " "lineno=1, col_offset=0, end_lineno=1, end_col_offset=24), " - "lineno=1, col_offset=0, end_lineno=1, end_col_offset=24)])", + "lineno=1, col_offset=0, end_lineno=1, end_col_offset=24)])" ) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_dump_indent(self): node = ast.parse('spam(eggs, "and cheese")') - self.assertEqual( - ast.dump(node, indent=3), - """\ + self.assertEqual(ast.dump(node, indent=3), """\ Module( body=[ Expr( @@ -1221,11 +1493,8 @@ def test_dump_indent(self): func=Name(id='spam', ctx=Load()), args=[ Name(id='eggs', ctx=Load()), - Constant(value='and cheese')]))])""", - ) - self.assertEqual( - ast.dump(node, annotate_fields=False, indent="\t"), - """\ + Constant(value='and cheese')]))])""") + self.assertEqual(ast.dump(node, annotate_fields=False, indent='\t'), """\ Module( \t[ \t\tExpr( @@ -1233,11 +1502,8 @@ def test_dump_indent(self): \t\t\t\tName('spam', Load()), \t\t\t\t[ \t\t\t\t\tName('eggs', Load()), -\t\t\t\t\tConstant('and cheese')]))])""", - ) - self.assertEqual( - ast.dump(node, include_attributes=True, indent=3), - """\ +\t\t\t\t\tConstant('and cheese')]))])""") + self.assertEqual(ast.dump(node, include_attributes=True, indent=3), """\ Module( body=[ Expr( @@ -1270,78 +1536,72 @@ def test_dump_indent(self): lineno=1, col_offset=0, end_lineno=1, - end_col_offset=24)])""", - ) + end_col_offset=24)])""") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_dump_incomplete(self): node = ast.Raise(lineno=3, col_offset=4) - self.assertEqual(ast.dump(node), "Raise()") - self.assertEqual( - ast.dump(node, include_attributes=True), "Raise(lineno=3, col_offset=4)" + self.assertEqual(ast.dump(node), + "Raise()" + ) + self.assertEqual(ast.dump(node, include_attributes=True), + "Raise(lineno=3, col_offset=4)" + ) + node = ast.Raise(exc=ast.Name(id='e', ctx=ast.Load()), lineno=3, col_offset=4) + self.assertEqual(ast.dump(node), + "Raise(exc=Name(id='e', ctx=Load()))" ) - node = ast.Raise(exc=ast.Name(id="e", ctx=ast.Load()), lineno=3, col_offset=4) - self.assertEqual(ast.dump(node), "Raise(exc=Name(id='e', ctx=Load()))") - self.assertEqual( - ast.dump(node, annotate_fields=False), "Raise(Name('e', Load()))" + self.assertEqual(ast.dump(node, annotate_fields=False), + "Raise(Name('e', Load()))" ) - self.assertEqual( - ast.dump(node, include_attributes=True), - "Raise(exc=Name(id='e', ctx=Load()), lineno=3, col_offset=4)", + self.assertEqual(ast.dump(node, include_attributes=True), + "Raise(exc=Name(id='e', ctx=Load()), lineno=3, col_offset=4)" ) - self.assertEqual( - ast.dump(node, annotate_fields=False, include_attributes=True), - "Raise(Name('e', Load()), lineno=3, col_offset=4)", + self.assertEqual(ast.dump(node, annotate_fields=False, include_attributes=True), + "Raise(Name('e', Load()), lineno=3, col_offset=4)" ) - node = ast.Raise(cause=ast.Name(id="e", ctx=ast.Load())) - self.assertEqual(ast.dump(node), "Raise(cause=Name(id='e', ctx=Load()))") - self.assertEqual( - ast.dump(node, annotate_fields=False), "Raise(cause=Name('e', Load()))" + node = ast.Raise(cause=ast.Name(id='e', ctx=ast.Load())) + self.assertEqual(ast.dump(node), + "Raise(cause=Name(id='e', ctx=Load()))" + ) + self.assertEqual(ast.dump(node, annotate_fields=False), + "Raise(cause=Name('e', Load()))" ) # Arguments: node = ast.arguments(args=[ast.arg("x")]) - self.assertEqual( - ast.dump(node, annotate_fields=False), + self.assertEqual(ast.dump(node, annotate_fields=False), "arguments([], [arg('x')])", ) node = ast.arguments(posonlyargs=[ast.arg("x")]) - self.assertEqual( - ast.dump(node, annotate_fields=False), + self.assertEqual(ast.dump(node, annotate_fields=False), "arguments([arg('x')])", ) - node = ast.arguments(posonlyargs=[ast.arg("x")], kwonlyargs=[ast.arg("y")]) - self.assertEqual( - ast.dump(node, annotate_fields=False), + node = ast.arguments(posonlyargs=[ast.arg("x")], kwonlyargs=[ast.arg('y')]) + self.assertEqual(ast.dump(node, annotate_fields=False), "arguments([arg('x')], kwonlyargs=[arg('y')])", ) - node = ast.arguments(args=[ast.arg("x")], kwonlyargs=[ast.arg("y")]) - self.assertEqual( - ast.dump(node, annotate_fields=False), + node = ast.arguments(args=[ast.arg("x")], kwonlyargs=[ast.arg('y')]) + self.assertEqual(ast.dump(node, annotate_fields=False), "arguments([], [arg('x')], kwonlyargs=[arg('y')])", ) node = ast.arguments() - self.assertEqual( - ast.dump(node, annotate_fields=False), + self.assertEqual(ast.dump(node, annotate_fields=False), "arguments()", ) # Classes: node = ast.ClassDef( - "T", + 'T', [], - [ast.keyword("a", ast.Constant(None))], + [ast.keyword('a', ast.Constant(None))], [], - [ast.Name("dataclass", ctx=ast.Load())], + [ast.Name('dataclass', ctx=ast.Load())], ) - self.assertEqual( - ast.dump(node), + self.assertEqual(ast.dump(node), "ClassDef(name='T', keywords=[keyword(arg='a', value=Constant(value=None))], decorator_list=[Name(id='dataclass', ctx=Load())])", ) - self.assertEqual( - ast.dump(node, annotate_fields=False), + self.assertEqual(ast.dump(node, annotate_fields=False), "ClassDef('T', [], [keyword('a', Constant(None))], [], [Name('dataclass', Load())])", ) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_dump_show_empty(self): def check_node(node, empty, full, **kwargs): with self.subTest(show_empty=False): @@ -1366,7 +1626,7 @@ def check_text(code, empty, full, **kwargs): check_node( # Corner case: there are no real `Name` instances with `id=''`: - ast.Name(id="", ctx=ast.Load()), + ast.Name(id='', ctx=ast.Load()), empty="Name(id='', ctx=Load())", full="Name(id='', ctx=Load())", ) @@ -1396,11 +1656,23 @@ def check_text(code, empty, full, **kwargs): ) check_node( - ast.Constant(value=""), + ast.Constant(value=''), empty="Constant(value='')", full="Constant(value='')", ) + check_node( + ast.Interpolation(value=ast.Constant(42), str=None, conversion=-1), + empty="Interpolation(value=Constant(value=42), str=None, conversion=-1)", + full="Interpolation(value=Constant(value=42), str=None, conversion=-1)", + ) + + check_node( + ast.Interpolation(value=ast.Constant(42), str=[], conversion=-1), + empty="Interpolation(value=Constant(value=42), str=[], conversion=-1)", + full="Interpolation(value=Constant(value=42), str=[], conversion=-1)", + ) + check_text( "def a(b: int = 0, *, c): ...", empty="Module(body=[FunctionDef(name='a', args=arguments(args=[arg(arg='b', annotation=Name(id='int', ctx=Load()))], kwonlyargs=[arg(arg='c')], kw_defaults=[None], defaults=[Constant(value=0)]), body=[Expr(value=Constant(value=Ellipsis))])])", @@ -1432,37 +1704,31 @@ def check_text(code, empty, full, **kwargs): full="Module(body=[Import(names=[alias(name='_ast', asname='ast')]), ImportFrom(module='module', names=[alias(name='sub')], level=0)], type_ignores=[])", ) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; ? ^^^^^^^^^ ^^^^^^^^^ def test_copy_location(self): - src = ast.parse("1 + 1", mode="eval") + src = ast.parse('1 + 1', mode='eval') src.body.right = ast.copy_location(ast.Constant(2), src.body.right) - self.assertEqual( - ast.dump(src, include_attributes=True), - "Expression(body=BinOp(left=Constant(value=1, lineno=1, col_offset=0, " - "end_lineno=1, end_col_offset=1), op=Add(), right=Constant(value=2, " - "lineno=1, col_offset=4, end_lineno=1, end_col_offset=5), lineno=1, " - "col_offset=0, end_lineno=1, end_col_offset=5))", - ) - func = ast.Name("spam", ast.Load()) - src = ast.Call( - col_offset=1, lineno=1, end_lineno=1, end_col_offset=1, func=func - ) + self.assertEqual(ast.dump(src, include_attributes=True), + 'Expression(body=BinOp(left=Constant(value=1, lineno=1, col_offset=0, ' + 'end_lineno=1, end_col_offset=1), op=Add(), right=Constant(value=2, ' + 'lineno=1, col_offset=4, end_lineno=1, end_col_offset=5), lineno=1, ' + 'col_offset=0, end_lineno=1, end_col_offset=5))' + ) + func = ast.Name('spam', ast.Load()) + src = ast.Call(col_offset=1, lineno=1, end_lineno=1, end_col_offset=1, func=func) new = ast.copy_location(src, ast.Call(col_offset=None, lineno=None, func=func)) self.assertIsNone(new.end_lineno) self.assertIsNone(new.end_col_offset) self.assertEqual(new.lineno, 1) self.assertEqual(new.col_offset, 1) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_fix_missing_locations(self): src = ast.parse('write("spam")') - src.body.append( - ast.Expr(ast.Call(ast.Name("spam", ast.Load()), [ast.Constant("eggs")], [])) - ) + src.body.append(ast.Expr(ast.Call(ast.Name('spam', ast.Load()), + [ast.Constant('eggs')], []))) self.assertEqual(src, ast.fix_missing_locations(src)) self.maxDiff = None - self.assertEqual( - ast.dump(src, include_attributes=True), + self.assertEqual(ast.dump(src, include_attributes=True), "Module(body=[Expr(value=Call(func=Name(id='write', ctx=Load(), " "lineno=1, col_offset=0, end_lineno=1, end_col_offset=5), " "args=[Constant(value='spam', lineno=1, col_offset=6, end_lineno=1, " @@ -1472,29 +1738,27 @@ def test_fix_missing_locations(self): "lineno=1, col_offset=0, end_lineno=1, end_col_offset=0), " "args=[Constant(value='eggs', lineno=1, col_offset=0, end_lineno=1, " "end_col_offset=0)], lineno=1, col_offset=0, end_lineno=1, " - "end_col_offset=0), lineno=1, col_offset=0, end_lineno=1, end_col_offset=0)])", + "end_col_offset=0), lineno=1, col_offset=0, end_lineno=1, end_col_offset=0)])" ) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; ? ^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ def test_increment_lineno(self): - src = ast.parse("1 + 1", mode="eval") + src = ast.parse('1 + 1', mode='eval') self.assertEqual(ast.increment_lineno(src, n=3), src) - self.assertEqual( - ast.dump(src, include_attributes=True), - "Expression(body=BinOp(left=Constant(value=1, lineno=4, col_offset=0, " - "end_lineno=4, end_col_offset=1), op=Add(), right=Constant(value=1, " - "lineno=4, col_offset=4, end_lineno=4, end_col_offset=5), lineno=4, " - "col_offset=0, end_lineno=4, end_col_offset=5))", + self.assertEqual(ast.dump(src, include_attributes=True), + 'Expression(body=BinOp(left=Constant(value=1, lineno=4, col_offset=0, ' + 'end_lineno=4, end_col_offset=1), op=Add(), right=Constant(value=1, ' + 'lineno=4, col_offset=4, end_lineno=4, end_col_offset=5), lineno=4, ' + 'col_offset=0, end_lineno=4, end_col_offset=5))' ) # issue10869: do not increment lineno of root twice - src = ast.parse("1 + 1", mode="eval") + src = ast.parse('1 + 1', mode='eval') self.assertEqual(ast.increment_lineno(src.body, n=3), src.body) - self.assertEqual( - ast.dump(src, include_attributes=True), - "Expression(body=BinOp(left=Constant(value=1, lineno=4, col_offset=0, " - "end_lineno=4, end_col_offset=1), op=Add(), right=Constant(value=1, " - "lineno=4, col_offset=4, end_lineno=4, end_col_offset=5), lineno=4, " - "col_offset=0, end_lineno=4, end_col_offset=5))", + self.assertEqual(ast.dump(src, include_attributes=True), + 'Expression(body=BinOp(left=Constant(value=1, lineno=4, col_offset=0, ' + 'end_lineno=4, end_col_offset=1), op=Add(), right=Constant(value=1, ' + 'lineno=4, col_offset=4, end_lineno=4, end_col_offset=5), lineno=4, ' + 'col_offset=0, end_lineno=4, end_col_offset=5))' ) src = ast.Call( func=ast.Name("test", ast.Load()), args=[], keywords=[], lineno=1 @@ -1502,85 +1766,82 @@ def test_increment_lineno(self): self.assertEqual(ast.increment_lineno(src).lineno, 2) self.assertIsNone(ast.increment_lineno(src).end_lineno) - @unittest.expectedFailure # TODO: RUSTPYTHON; IndexError: index out of range def test_increment_lineno_on_module(self): - src = ast.parse( - dedent("""\ + src = ast.parse(dedent("""\ a = 1 b = 2 # type: ignore c = 3 d = 4 # type: ignore@tag - """), - type_comments=True, - ) + """), type_comments=True) ast.increment_lineno(src, n=5) self.assertEqual(src.type_ignores[0].lineno, 7) self.assertEqual(src.type_ignores[1].lineno, 9) - self.assertEqual(src.type_ignores[1].tag, "@tag") + self.assertEqual(src.type_ignores[1].tag, '@tag') def test_iter_fields(self): - node = ast.parse("foo()", mode="eval") + node = ast.parse('foo()', mode='eval') d = dict(ast.iter_fields(node.body)) - self.assertEqual(d.pop("func").id, "foo") - self.assertEqual(d, {"keywords": [], "args": []}) + self.assertEqual(d.pop('func').id, 'foo') + self.assertEqual(d, {'keywords': [], 'args': []}) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_iter_child_nodes(self): - node = ast.parse("spam(23, 42, eggs='leek')", mode="eval") + node = ast.parse("spam(23, 42, eggs='leek')", mode='eval') self.assertEqual(len(list(ast.iter_child_nodes(node.body))), 4) iterator = ast.iter_child_nodes(node.body) - self.assertEqual(next(iterator).id, "spam") + self.assertEqual(next(iterator).id, 'spam') self.assertEqual(next(iterator).value, 23) self.assertEqual(next(iterator).value, 42) - self.assertEqual( - ast.dump(next(iterator)), - "keyword(arg='eggs', value=Constant(value='leek'))", + self.assertEqual(ast.dump(next(iterator)), + "keyword(arg='eggs', value=Constant(value='leek'))" ) def test_get_docstring(self): node = ast.parse('"""line one\n line two"""') - self.assertEqual(ast.get_docstring(node), "line one\nline two") + self.assertEqual(ast.get_docstring(node), + 'line one\nline two') node = ast.parse('class foo:\n """line one\n line two"""') - self.assertEqual(ast.get_docstring(node.body[0]), "line one\nline two") + self.assertEqual(ast.get_docstring(node.body[0]), + 'line one\nline two') node = ast.parse('def foo():\n """line one\n line two"""') - self.assertEqual(ast.get_docstring(node.body[0]), "line one\nline two") + self.assertEqual(ast.get_docstring(node.body[0]), + 'line one\nline two') node = ast.parse('async def foo():\n """spam\n ham"""') - self.assertEqual(ast.get_docstring(node.body[0]), "spam\nham") + self.assertEqual(ast.get_docstring(node.body[0]), 'spam\nham') node = ast.parse('async def foo():\n """spam\n ham"""') - self.assertEqual(ast.get_docstring(node.body[0], clean=False), "spam\n ham") + self.assertEqual(ast.get_docstring(node.body[0], clean=False), 'spam\n ham') - node = ast.parse("x") + node = ast.parse('x') self.assertRaises(TypeError, ast.get_docstring, node.body[0]) def test_get_docstring_none(self): - self.assertIsNone(ast.get_docstring(ast.parse(""))) + self.assertIsNone(ast.get_docstring(ast.parse(''))) node = ast.parse('x = "not docstring"') self.assertIsNone(ast.get_docstring(node)) - node = ast.parse("def foo():\n pass") + node = ast.parse('def foo():\n pass') self.assertIsNone(ast.get_docstring(node)) - node = ast.parse("class foo:\n pass") + node = ast.parse('class foo:\n pass') self.assertIsNone(ast.get_docstring(node.body[0])) node = ast.parse('class foo:\n x = "not docstring"') self.assertIsNone(ast.get_docstring(node.body[0])) - node = ast.parse("class foo:\n def bar(self): pass") + node = ast.parse('class foo:\n def bar(self): pass') self.assertIsNone(ast.get_docstring(node.body[0])) - node = ast.parse("def foo():\n pass") + node = ast.parse('def foo():\n pass') self.assertIsNone(ast.get_docstring(node.body[0])) node = ast.parse('def foo():\n x = "not docstring"') self.assertIsNone(ast.get_docstring(node.body[0])) - node = ast.parse("async def foo():\n pass") + node = ast.parse('async def foo():\n pass') self.assertIsNone(ast.get_docstring(node.body[0])) node = ast.parse('async def foo():\n x = "not docstring"') self.assertIsNone(ast.get_docstring(node.body[0])) - node = ast.parse("async def foo():\n 42") + node = ast.parse('async def foo():\n 42') self.assertIsNone(ast.get_docstring(node.body[0])) def test_multi_line_docstring_col_offset_and_lineno_issue16806(self): @@ -1603,83 +1864,79 @@ def test_multi_line_docstring_col_offset_and_lineno_issue16806(self): self.assertEqual(node.body[2].lineno, 13) def test_elif_stmt_start_position(self): - node = ast.parse("if a:\n pass\nelif b:\n pass\n") + node = ast.parse('if a:\n pass\nelif b:\n pass\n') elif_stmt = node.body[0].orelse[0] self.assertEqual(elif_stmt.lineno, 3) self.assertEqual(elif_stmt.col_offset, 0) def test_elif_stmt_start_position_with_else(self): - node = ast.parse("if a:\n pass\nelif b:\n pass\nelse:\n pass\n") + node = ast.parse('if a:\n pass\nelif b:\n pass\nelse:\n pass\n') elif_stmt = node.body[0].orelse[0] self.assertEqual(elif_stmt.lineno, 3) self.assertEqual(elif_stmt.col_offset, 0) def test_starred_expr_end_position_within_call(self): - node = ast.parse("f(*[0, 1])") + node = ast.parse('f(*[0, 1])') starred_expr = node.body[0].value.args[0] self.assertEqual(starred_expr.end_lineno, 1) self.assertEqual(starred_expr.end_col_offset, 9) def test_literal_eval(self): - self.assertEqual(ast.literal_eval("[1, 2, 3]"), [1, 2, 3]) + self.assertEqual(ast.literal_eval('[1, 2, 3]'), [1, 2, 3]) self.assertEqual(ast.literal_eval('{"foo": 42}'), {"foo": 42}) - self.assertEqual(ast.literal_eval("(True, False, None)"), (True, False, None)) - self.assertEqual(ast.literal_eval("{1, 2, 3}"), {1, 2, 3}) + self.assertEqual(ast.literal_eval('(True, False, None)'), (True, False, None)) + self.assertEqual(ast.literal_eval('{1, 2, 3}'), {1, 2, 3}) self.assertEqual(ast.literal_eval('b"hi"'), b"hi") - self.assertEqual(ast.literal_eval("set()"), set()) - self.assertRaises(ValueError, ast.literal_eval, "foo()") - self.assertEqual(ast.literal_eval("6"), 6) - self.assertEqual(ast.literal_eval("+6"), 6) - self.assertEqual(ast.literal_eval("-6"), -6) - self.assertEqual(ast.literal_eval("3.25"), 3.25) - self.assertEqual(ast.literal_eval("+3.25"), 3.25) - self.assertEqual(ast.literal_eval("-3.25"), -3.25) - self.assertEqual(repr(ast.literal_eval("-0.0")), "-0.0") - self.assertRaises(ValueError, ast.literal_eval, "++6") - self.assertRaises(ValueError, ast.literal_eval, "+True") - self.assertRaises(ValueError, ast.literal_eval, "2+3") - - @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError not raised + self.assertEqual(ast.literal_eval('set()'), set()) + self.assertRaises(ValueError, ast.literal_eval, 'foo()') + self.assertEqual(ast.literal_eval('6'), 6) + self.assertEqual(ast.literal_eval('+6'), 6) + self.assertEqual(ast.literal_eval('-6'), -6) + self.assertEqual(ast.literal_eval('3.25'), 3.25) + self.assertEqual(ast.literal_eval('+3.25'), 3.25) + self.assertEqual(ast.literal_eval('-3.25'), -3.25) + self.assertEqual(repr(ast.literal_eval('-0.0')), '-0.0') + self.assertRaises(ValueError, ast.literal_eval, '++6') + self.assertRaises(ValueError, ast.literal_eval, '+True') + self.assertRaises(ValueError, ast.literal_eval, '2+3') + + @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError not raised def test_literal_eval_str_int_limit(self): with support.adjust_int_max_str_digits(4000): - ast.literal_eval("3" * 4000) # no error + ast.literal_eval('3'*4000) # no error with self.assertRaises(SyntaxError) as err_ctx: - ast.literal_eval("3" * 4001) - self.assertIn("Exceeds the limit ", str(err_ctx.exception)) - self.assertIn(" Consider hexadecimal ", str(err_ctx.exception)) + ast.literal_eval('3'*4001) + self.assertIn('Exceeds the limit ', str(err_ctx.exception)) + self.assertIn(' Consider hexadecimal ', str(err_ctx.exception)) def test_literal_eval_complex(self): # Issue #4907 - self.assertEqual(ast.literal_eval("6j"), 6j) - self.assertEqual(ast.literal_eval("-6j"), -6j) - self.assertEqual(ast.literal_eval("6.75j"), 6.75j) - self.assertEqual(ast.literal_eval("-6.75j"), -6.75j) - self.assertEqual(ast.literal_eval("3+6j"), 3 + 6j) - self.assertEqual(ast.literal_eval("-3+6j"), -3 + 6j) - self.assertEqual(ast.literal_eval("3-6j"), 3 - 6j) - self.assertEqual(ast.literal_eval("-3-6j"), -3 - 6j) - self.assertEqual(ast.literal_eval("3.25+6.75j"), 3.25 + 6.75j) - self.assertEqual(ast.literal_eval("-3.25+6.75j"), -3.25 + 6.75j) - self.assertEqual(ast.literal_eval("3.25-6.75j"), 3.25 - 6.75j) - self.assertEqual(ast.literal_eval("-3.25-6.75j"), -3.25 - 6.75j) - self.assertEqual(ast.literal_eval("(3+6j)"), 3 + 6j) - self.assertRaises(ValueError, ast.literal_eval, "-6j+3") - self.assertRaises(ValueError, ast.literal_eval, "-6j+3j") - self.assertRaises(ValueError, ast.literal_eval, "3+-6j") - self.assertRaises(ValueError, ast.literal_eval, "3+(0+6j)") - self.assertRaises(ValueError, ast.literal_eval, "-(3+6j)") + self.assertEqual(ast.literal_eval('6j'), 6j) + self.assertEqual(ast.literal_eval('-6j'), -6j) + self.assertEqual(ast.literal_eval('6.75j'), 6.75j) + self.assertEqual(ast.literal_eval('-6.75j'), -6.75j) + self.assertEqual(ast.literal_eval('3+6j'), 3+6j) + self.assertEqual(ast.literal_eval('-3+6j'), -3+6j) + self.assertEqual(ast.literal_eval('3-6j'), 3-6j) + self.assertEqual(ast.literal_eval('-3-6j'), -3-6j) + self.assertEqual(ast.literal_eval('3.25+6.75j'), 3.25+6.75j) + self.assertEqual(ast.literal_eval('-3.25+6.75j'), -3.25+6.75j) + self.assertEqual(ast.literal_eval('3.25-6.75j'), 3.25-6.75j) + self.assertEqual(ast.literal_eval('-3.25-6.75j'), -3.25-6.75j) + self.assertEqual(ast.literal_eval('(3+6j)'), 3+6j) + self.assertRaises(ValueError, ast.literal_eval, '-6j+3') + self.assertRaises(ValueError, ast.literal_eval, '-6j+3j') + self.assertRaises(ValueError, ast.literal_eval, '3+-6j') + self.assertRaises(ValueError, ast.literal_eval, '3+(0+6j)') + self.assertRaises(ValueError, ast.literal_eval, '-(3+6j)') def test_literal_eval_malformed_dict_nodes(self): - malformed = ast.Dict( - keys=[ast.Constant(1), ast.Constant(2)], values=[ast.Constant(3)] - ) + malformed = ast.Dict(keys=[ast.Constant(1), ast.Constant(2)], values=[ast.Constant(3)]) self.assertRaises(ValueError, ast.literal_eval, malformed) - malformed = ast.Dict( - keys=[ast.Constant(1)], values=[ast.Constant(2), ast.Constant(3)] - ) + malformed = ast.Dict(keys=[ast.Constant(1)], values=[ast.Constant(2), ast.Constant(3)]) self.assertRaises(ValueError, ast.literal_eval, malformed) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError: expected an expression def test_literal_eval_trailing_ws(self): self.assertEqual(ast.literal_eval(" -1"), -1) self.assertEqual(ast.literal_eval("\t\t-1"), -1) @@ -1687,58 +1944,52 @@ def test_literal_eval_trailing_ws(self): self.assertRaises(IndentationError, ast.literal_eval, "\n -1") def test_literal_eval_malformed_lineno(self): - msg = r"malformed node or string on line 3:" + msg = r'malformed node or string on line 3:' with self.assertRaisesRegex(ValueError, msg): ast.literal_eval("{'a': 1,\n'b':2,\n'c':++3,\n'd':4}") - node = ast.UnaryOp(ast.UAdd(), ast.UnaryOp(ast.UAdd(), ast.Constant(6))) - self.assertIsNone(getattr(node, "lineno", None)) - msg = r"malformed node or string:" + node = ast.UnaryOp( + ast.UAdd(), ast.UnaryOp(ast.UAdd(), ast.Constant(6))) + self.assertIsNone(getattr(node, 'lineno', None)) + msg = r'malformed node or string:' with self.assertRaisesRegex(ValueError, msg): ast.literal_eval(node) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: "unexpected indent" does not match "expected an expression (, line 2)" def test_literal_eval_syntax_errors(self): with self.assertRaisesRegex(SyntaxError, "unexpected indent"): - ast.literal_eval(r""" + ast.literal_eval(r''' \ (\ - \ """) + \ ''') - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: required field "lineno" missing from alias def test_bad_integer(self): # issue13436: Bad error message with invalid numeric values - body = [ - ast.ImportFrom( - module="time", - names=[ast.alias(name="sleep")], - level=None, - lineno=None, - col_offset=None, - ) - ] + body = [ast.ImportFrom(module='time', + names=[ast.alias(name='sleep')], + level=None, + lineno=None, col_offset=None)] mod = ast.Module(body, []) with self.assertRaises(ValueError) as cm: - compile(mod, "test", "exec") + compile(mod, 'test', 'exec') self.assertIn("invalid integer value: None", str(cm.exception)) def test_level_as_none(self): - body = [ - ast.ImportFrom( - module="time", - names=[ast.alias(name="sleep", lineno=0, col_offset=0)], - level=None, - lineno=0, - col_offset=0, - ) - ] + body = [ast.ImportFrom(module='time', + names=[ast.alias(name='sleep', + lineno=0, col_offset=0)], + level=None, + lineno=0, col_offset=0)] mod = ast.Module(body, []) - code = compile(mod, "test", "exec") + code = compile(mod, 'test', 'exec') ns = {} exec(code, ns) - self.assertIn("sleep", ns) + self.assertIn('sleep', ns) - @unittest.skip('TODO: RUSTPYTHON; crash') + @unittest.skip("TODO: RUSTPYTHON; crash") + @skip_if_unlimited_stack_size + @skip_emscripten_stack_overflow() def test_recursion_direct(self): e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0, operand=ast.Constant(1)) e.operand = e @@ -1746,7 +1997,9 @@ def test_recursion_direct(self): with support.infinite_recursion(): compile(ast.Expression(e), "", "eval") - @unittest.skip('TODO: RUSTPYTHON; crash') + @unittest.skip("TODO: RUSTPYTHON; crash") + @skip_if_unlimited_stack_size + @skip_emscripten_stack_overflow() def test_recursion_indirect(self): e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0, operand=ast.Constant(1)) f = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0, operand=ast.Constant(1)) @@ -1758,6 +2011,7 @@ def test_recursion_indirect(self): class ASTValidatorTests(unittest.TestCase): + def mod(self, mod, msg=None, mode="exec", *, exc=ValueError): mod.lineno = mod.col_offset = 0 ast.fix_missing_locations(mod) @@ -1776,7 +2030,6 @@ def stmt(self, stmt, msg=None): mod = ast.Module([stmt], []) self.mod(mod, msg) - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_module(self): m = ast.Interactive([ast.Expr(ast.Name("x", ast.Store()))]) self.mod(m, "must have Load context", "single") @@ -1784,15 +2037,9 @@ def test_module(self): self.mod(m, "must have Load context", "eval") def _check_arguments(self, fac, check): - def arguments( - args=None, - posonlyargs=None, - vararg=None, - kwonlyargs=None, - kwarg=None, - defaults=None, - kw_defaults=None, - ): + def arguments(args=None, posonlyargs=None, vararg=None, + kwonlyargs=None, kwarg=None, + defaults=None, kw_defaults=None): if args is None: args = [] if posonlyargs is None: @@ -1803,69 +2050,50 @@ def arguments( defaults = [] if kw_defaults is None: kw_defaults = [] - args = ast.arguments( - args, posonlyargs, vararg, kwonlyargs, kw_defaults, kwarg, defaults - ) + args = ast.arguments(args, posonlyargs, vararg, kwonlyargs, + kw_defaults, kwarg, defaults) return fac(args) - args = [ast.arg("x", ast.Name("x", ast.Store()))] check(arguments(args=args), "must have Load context") check(arguments(posonlyargs=args), "must have Load context") check(arguments(kwonlyargs=args), "must have Load context") - check( - arguments(defaults=[ast.Constant(3)]), "more positional defaults than args" - ) - check( - arguments(kw_defaults=[ast.Constant(4)]), - "length of kwonlyargs is not the same as kw_defaults", - ) + check(arguments(defaults=[ast.Constant(3)]), + "more positional defaults than args") + check(arguments(kw_defaults=[ast.Constant(4)]), + "length of kwonlyargs is not the same as kw_defaults") args = [ast.arg("x", ast.Name("x", ast.Load()))] - check( - arguments(args=args, defaults=[ast.Name("x", ast.Store())]), - "must have Load context", - ) - args = [ - ast.arg("a", ast.Name("x", ast.Load())), - ast.arg("b", ast.Name("y", ast.Load())), - ] - check( - arguments(kwonlyargs=args, kw_defaults=[None, ast.Name("x", ast.Store())]), - "must have Load context", - ) - - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + check(arguments(args=args, defaults=[ast.Name("x", ast.Store())]), + "must have Load context") + args = [ast.arg("a", ast.Name("x", ast.Load())), + ast.arg("b", ast.Name("y", ast.Load()))] + check(arguments(kwonlyargs=args, + kw_defaults=[None, ast.Name("x", ast.Store())]), + "must have Load context") + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_funcdef(self): a = ast.arguments([], [], None, [], [], None, []) f = ast.FunctionDef("x", a, [], [], None, None, []) self.stmt(f, "empty body on FunctionDef") - f = ast.FunctionDef( - "x", a, [ast.Pass()], [ast.Name("x", ast.Store())], None, None, [] - ) + f = ast.FunctionDef("x", a, [ast.Pass()], [ast.Name("x", ast.Store())], None, None, []) self.stmt(f, "must have Load context") - f = ast.FunctionDef( - "x", a, [ast.Pass()], [], ast.Name("x", ast.Store()), None, [] - ) + f = ast.FunctionDef("x", a, [ast.Pass()], [], + ast.Name("x", ast.Store()), None, []) self.stmt(f, "must have Load context") f = ast.FunctionDef("x", ast.arguments(), [ast.Pass()]) self.stmt(f) - def fac(args): return ast.FunctionDef("x", args, [ast.Pass()], [], None, None, []) - self._check_arguments(fac, self.stmt) - @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: class pattern defines no positional sub-patterns (__match_args__ missing) def test_funcdef_pattern_matching(self): # gh-104799: New fields on FunctionDef should be added at the end def matcher(node): match node: - case ast.FunctionDef( - "foo", - ast.arguments(args=[ast.arg("bar")]), - [ast.Pass()], - [ast.Name("capybara", ast.Load())], - ast.Name("pacarana", ast.Load()), - ): + case ast.FunctionDef("foo", ast.arguments(args=[ast.arg("bar")]), + [ast.Pass()], + [ast.Name("capybara", ast.Load())], + ast.Name("pacarana", ast.Load())): return True case _: return False @@ -1880,11 +2108,9 @@ def foo(bar) -> pacarana: self.assertIsInstance(funcdef, ast.FunctionDef) self.assertTrue(matcher(funcdef)) - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_classdef(self): - def cls( - bases=None, keywords=None, body=None, decorator_list=None, type_params=None - ): + def cls(bases=None, keywords=None, body=None, decorator_list=None, type_params=None): if bases is None: bases = [] if keywords is None: @@ -1895,94 +2121,73 @@ def cls( decorator_list = [] if type_params is None: type_params = [] - return ast.ClassDef( - "myclass", bases, keywords, body, decorator_list, type_params - ) - - self.stmt(cls(bases=[ast.Name("x", ast.Store())]), "must have Load context") - self.stmt( - cls(keywords=[ast.keyword("x", ast.Name("x", ast.Store()))]), - "must have Load context", - ) + return ast.ClassDef("myclass", bases, keywords, + body, decorator_list, type_params) + self.stmt(cls(bases=[ast.Name("x", ast.Store())]), + "must have Load context") + self.stmt(cls(keywords=[ast.keyword("x", ast.Name("x", ast.Store()))]), + "must have Load context") self.stmt(cls(body=[]), "empty body on ClassDef") self.stmt(cls(body=[None]), "None disallowed") - self.stmt( - cls(decorator_list=[ast.Name("x", ast.Store())]), "must have Load context" - ) + self.stmt(cls(decorator_list=[ast.Name("x", ast.Store())]), + "must have Load context") - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_delete(self): self.stmt(ast.Delete([]), "empty targets on Delete") self.stmt(ast.Delete([None]), "None disallowed") - self.stmt(ast.Delete([ast.Name("x", ast.Load())]), "must have Del context") + self.stmt(ast.Delete([ast.Name("x", ast.Load())]), + "must have Del context") - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_assign(self): self.stmt(ast.Assign([], ast.Constant(3)), "empty targets on Assign") self.stmt(ast.Assign([None], ast.Constant(3)), "None disallowed") - self.stmt( - ast.Assign([ast.Name("x", ast.Load())], ast.Constant(3)), - "must have Store context", - ) - self.stmt( - ast.Assign([ast.Name("x", ast.Store())], ast.Name("y", ast.Store())), - "must have Load context", - ) + self.stmt(ast.Assign([ast.Name("x", ast.Load())], ast.Constant(3)), + "must have Store context") + self.stmt(ast.Assign([ast.Name("x", ast.Store())], + ast.Name("y", ast.Store())), + "must have Load context") - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_augassign(self): - aug = ast.AugAssign( - ast.Name("x", ast.Load()), ast.Add(), ast.Name("y", ast.Load()) - ) + aug = ast.AugAssign(ast.Name("x", ast.Load()), ast.Add(), + ast.Name("y", ast.Load())) self.stmt(aug, "must have Store context") - aug = ast.AugAssign( - ast.Name("x", ast.Store()), ast.Add(), ast.Name("y", ast.Store()) - ) + aug = ast.AugAssign(ast.Name("x", ast.Store()), ast.Add(), + ast.Name("y", ast.Store())) self.stmt(aug, "must have Load context") - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_for(self): x = ast.Name("x", ast.Store()) y = ast.Name("y", ast.Load()) p = ast.Pass() self.stmt(ast.For(x, y, [], []), "empty body on For") - self.stmt( - ast.For(ast.Name("x", ast.Load()), y, [p], []), "must have Store context" - ) - self.stmt( - ast.For(x, ast.Name("y", ast.Store()), [p], []), "must have Load context" - ) + self.stmt(ast.For(ast.Name("x", ast.Load()), y, [p], []), + "must have Store context") + self.stmt(ast.For(x, ast.Name("y", ast.Store()), [p], []), + "must have Load context") e = ast.Expr(ast.Name("x", ast.Store())) self.stmt(ast.For(x, y, [e], []), "must have Load context") self.stmt(ast.For(x, y, [p], [e]), "must have Load context") - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_while(self): self.stmt(ast.While(ast.Constant(3), [], []), "empty body on While") - self.stmt( - ast.While(ast.Name("x", ast.Store()), [ast.Pass()], []), - "must have Load context", - ) - self.stmt( - ast.While( - ast.Constant(3), [ast.Pass()], [ast.Expr(ast.Name("x", ast.Store()))] - ), - "must have Load context", - ) + self.stmt(ast.While(ast.Name("x", ast.Store()), [ast.Pass()], []), + "must have Load context") + self.stmt(ast.While(ast.Constant(3), [ast.Pass()], + [ast.Expr(ast.Name("x", ast.Store()))]), + "must have Load context") - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_if(self): self.stmt(ast.If(ast.Constant(3), [], []), "empty body on If") i = ast.If(ast.Name("x", ast.Store()), [ast.Pass()], []) self.stmt(i, "must have Load context") i = ast.If(ast.Constant(3), [ast.Expr(ast.Name("x", ast.Store()))], []) self.stmt(i, "must have Load context") - i = ast.If( - ast.Constant(3), [ast.Pass()], [ast.Expr(ast.Name("x", ast.Store()))] - ) + i = ast.If(ast.Constant(3), [ast.Pass()], + [ast.Expr(ast.Name("x", ast.Store()))]) self.stmt(i, "must have Load context") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_with(self): p = ast.Pass() self.stmt(ast.With([], [p]), "empty items on With") @@ -1993,7 +2198,6 @@ def test_with(self): i = ast.withitem(ast.Constant(3), ast.Name("x", ast.Load())) self.stmt(ast.With([i], [p]), "must have Store context") - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_raise(self): r = ast.Raise(None, ast.Constant(3)) self.stmt(r, "Raise with cause but no exception") @@ -2002,7 +2206,6 @@ def test_raise(self): r = ast.Raise(ast.Constant(4), ast.Name("x", ast.Store())) self.stmt(r, "must have Load context") - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_try(self): p = ast.Pass() t = ast.Try([], [], [], [p]) @@ -2023,7 +2226,6 @@ def test_try(self): t = ast.Try([p], e, [p], [ast.Expr(ast.Name("x", ast.Store()))]) self.stmt(t, "must have Load context") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_try_star(self): p = ast.Pass() t = ast.TryStar([], [], [], [p]) @@ -2044,38 +2246,32 @@ def test_try_star(self): t = ast.TryStar([p], e, [p], [ast.Expr(ast.Name("x", ast.Store()))]) self.stmt(t, "must have Load context") - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_assert(self): - self.stmt( - ast.Assert(ast.Name("x", ast.Store()), None), "must have Load context" - ) - assrt = ast.Assert(ast.Name("x", ast.Load()), ast.Name("y", ast.Store())) + self.stmt(ast.Assert(ast.Name("x", ast.Store()), None), + "must have Load context") + assrt = ast.Assert(ast.Name("x", ast.Load()), + ast.Name("y", ast.Store())) self.stmt(assrt, "must have Load context") - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_import(self): self.stmt(ast.Import([]), "empty names on Import") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_importfrom(self): imp = ast.ImportFrom(None, [ast.alias("x", None)], -42) self.stmt(imp, "Negative ImportFrom level") self.stmt(ast.ImportFrom(None, [], 0), "empty names on ImportFrom") - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_global(self): self.stmt(ast.Global([]), "empty names on Global") - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_nonlocal(self): self.stmt(ast.Nonlocal([]), "empty names on Nonlocal") - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_expr(self): e = ast.Expr(ast.Name("x", ast.Store())) self.stmt(e, "must have Load context") - @unittest.skip('TODO: RUSTPYTHON; called `Option::unwrap()` on a `None` value') + @unittest.skip("TODO: RUSTPYTHON; called `Option::unwrap()` on a `None` value") def test_boolop(self): b = ast.BoolOp(ast.And(), []) self.expr(b, "less than 2 values") @@ -2086,36 +2282,33 @@ def test_boolop(self): b = ast.BoolOp(ast.And(), [ast.Constant(4), ast.Name("x", ast.Store())]) self.expr(b, "must have Load context") - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_unaryop(self): u = ast.UnaryOp(ast.Not(), ast.Name("x", ast.Store())) self.expr(u, "must have Load context") - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_lambda(self): a = ast.arguments([], [], None, [], [], None, []) - self.expr(ast.Lambda(a, ast.Name("x", ast.Store())), "must have Load context") - + self.expr(ast.Lambda(a, ast.Name("x", ast.Store())), + "must have Load context") def fac(args): return ast.Lambda(args, ast.Name("x", ast.Load())) - self._check_arguments(fac, self.expr) - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_ifexp(self): l = ast.Name("x", ast.Load()) s = ast.Name("y", ast.Store()) for args in (s, l, l), (l, s, l), (l, l, s): self.expr(ast.IfExp(*args), "must have Load context") - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_dict(self): d = ast.Dict([], [ast.Name("x", ast.Load())]) self.expr(d, "same number of keys as values") d = ast.Dict([ast.Name("x", ast.Load())], [None]) self.expr(d, "None disallowed") - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: expected some sort of expr, but got None def test_set(self): self.expr(ast.Set([None]), "None disallowed") s = ast.Set([ast.Name("x", ast.Store())]) @@ -2123,13 +2316,11 @@ def test_set(self): def _check_comprehension(self, fac): self.expr(fac([]), "comprehension with no generators") - g = ast.comprehension( - ast.Name("x", ast.Load()), ast.Name("x", ast.Load()), [], 0 - ) + g = ast.comprehension(ast.Name("x", ast.Load()), + ast.Name("x", ast.Load()), [], 0) self.expr(fac([g]), "must have Store context") - g = ast.comprehension( - ast.Name("x", ast.Store()), ast.Name("x", ast.Store()), [], 0 - ) + g = ast.comprehension(ast.Name("x", ast.Store()), + ast.Name("x", ast.Store()), [], 0) self.expr(fac([g]), "must have Load context") x = ast.Name("x", ast.Store()) y = ast.Name("y", ast.Load()) @@ -2139,46 +2330,42 @@ def _check_comprehension(self, fac): self.expr(fac([g]), "must have Load context") def _simple_comp(self, fac): - g = ast.comprehension( - ast.Name("x", ast.Store()), ast.Name("x", ast.Load()), [], 0 - ) - self.expr(fac(ast.Name("x", ast.Store()), [g]), "must have Load context") - + g = ast.comprehension(ast.Name("x", ast.Store()), + ast.Name("x", ast.Load()), [], 0) + self.expr(fac(ast.Name("x", ast.Store()), [g]), + "must have Load context") def wrap(gens): return fac(ast.Name("x", ast.Store()), gens) - self._check_comprehension(wrap) - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_listcomp(self): self._simple_comp(ast.ListComp) - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_setcomp(self): self._simple_comp(ast.SetComp) - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_generatorexp(self): self._simple_comp(ast.GeneratorExp) - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_dictcomp(self): - g = ast.comprehension( - ast.Name("y", ast.Store()), ast.Name("p", ast.Load()), [], 0 - ) - c = ast.DictComp(ast.Name("x", ast.Store()), ast.Name("y", ast.Load()), [g]) + g = ast.comprehension(ast.Name("y", ast.Store()), + ast.Name("p", ast.Load()), [], 0) + c = ast.DictComp(ast.Name("x", ast.Store()), + ast.Name("y", ast.Load()), [g]) self.expr(c, "must have Load context") - c = ast.DictComp(ast.Name("x", ast.Load()), ast.Name("y", ast.Store()), [g]) + c = ast.DictComp(ast.Name("x", ast.Load()), + ast.Name("y", ast.Store()), [g]) self.expr(c, "must have Load context") - def factory(comps): k = ast.Name("x", ast.Load()) v = ast.Name("y", ast.Load()) return ast.DictComp(k, v, comps) - self._check_comprehension(factory) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_yield(self): self.expr(ast.Yield(ast.Name("x", ast.Store())), "must have Load") self.expr(ast.YieldFrom(ast.Name("x", ast.Store())), "must have Load") @@ -2195,7 +2382,7 @@ def test_compare(self): comp = ast.Compare(left, [ast.In()], [ast.Constant("blah")]) self.expr(comp) - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_call(self): func = ast.Name("x", ast.Load()) args = [ast.Name("y", ast.Load())] @@ -2208,202 +2395,204 @@ def test_call(self): call = ast.Call(func, args, bad_keywords) self.expr(call, "must have Load context") - def test_num(self): - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings("ignore", "", DeprecationWarning) - from ast import Num - - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings("always", "", DeprecationWarning) - - class subint(int): - pass - - class subfloat(float): - pass - - class subcomplex(complex): - pass - - for obj in "0", "hello": - self.expr(ast.Num(obj)) - for obj in subint(), subfloat(), subcomplex(): - self.expr(ast.Num(obj), "invalid type", exc=TypeError) - - self.assertEqual( - [str(w.message) for w in wlog], - [ - "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", - "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", - "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", - "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", - "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", - ], - ) - - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_attribute(self): attr = ast.Attribute(ast.Name("x", ast.Store()), "y", ast.Load()) self.expr(attr, "must have Load context") - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_subscript(self): - sub = ast.Subscript(ast.Name("x", ast.Store()), ast.Constant(3), ast.Load()) + sub = ast.Subscript(ast.Name("x", ast.Store()), ast.Constant(3), + ast.Load()) self.expr(sub, "must have Load context") x = ast.Name("x", ast.Load()) - sub = ast.Subscript(x, ast.Name("y", ast.Store()), ast.Load()) + sub = ast.Subscript(x, ast.Name("y", ast.Store()), + ast.Load()) self.expr(sub, "must have Load context") s = ast.Name("x", ast.Store()) for args in (s, None, None), (None, s, None), (None, None, s): sl = ast.Slice(*args) - self.expr(ast.Subscript(x, sl, ast.Load()), "must have Load context") + self.expr(ast.Subscript(x, sl, ast.Load()), + "must have Load context") sl = ast.Tuple([], ast.Load()) self.expr(ast.Subscript(x, sl, ast.Load())) sl = ast.Tuple([s], ast.Load()) self.expr(ast.Subscript(x, sl, ast.Load()), "must have Load context") - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_starred(self): - left = ast.List( - [ast.Starred(ast.Name("x", ast.Load()), ast.Store())], ast.Store() - ) + left = ast.List([ast.Starred(ast.Name("x", ast.Load()), ast.Store())], + ast.Store()) assign = ast.Assign([left], ast.Constant(4)) self.stmt(assign, "must have Store context") def _sequence(self, fac): self.expr(fac([None], ast.Load()), "None disallowed") - self.expr( - fac([ast.Name("x", ast.Store())], ast.Load()), "must have Load context" - ) + self.expr(fac([ast.Name("x", ast.Store())], ast.Load()), + "must have Load context") - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: expected some sort of expr, but got None def test_list(self): self._sequence(ast.List) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: expected some sort of expr, but got None def test_tuple(self): self._sequence(ast.Tuple) - def test_nameconstant(self): - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings("ignore", "", DeprecationWarning) - from ast import NameConstant - - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings("always", "", DeprecationWarning) - self.expr(ast.NameConstant(4)) - - self.assertEqual( - [str(w.message) for w in wlog], - [ - "ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead", - ], - ) - - @unittest.expectedFailure # TODO: RUSTPYTHON - @support.requires_resource("cpu") + @support.requires_resource('cpu') def test_stdlib_validates(self): - stdlib = os.path.dirname(ast.__file__) - tests = [fn for fn in os.listdir(stdlib) if fn.endswith(".py")] - tests.extend(["test/test_grammar.py", "test/test_unpack_ex.py"]) - for module in tests: + for module in STDLIB_FILES: with self.subTest(module): - fn = os.path.join(stdlib, module) + fn = os.path.join(STDLIB, module) with open(fn, "r", encoding="utf-8") as fp: source = fp.read() mod = ast.parse(source, fn) compile(mod, fn, "exec") + mod2 = ast.parse(source, fn) + self.assertTrue(ast.compare(mod, mod2)) constant_1 = ast.Constant(1) pattern_1 = ast.MatchValue(constant_1) - constant_x = ast.Constant("x") + constant_x = ast.Constant('x') pattern_x = ast.MatchValue(constant_x) constant_true = ast.Constant(True) pattern_true = ast.MatchSingleton(True) - name_carter = ast.Name("carter", ast.Load()) + name_carter = ast.Name('carter', ast.Load()) _MATCH_PATTERNS = [ ast.MatchValue( ast.Attribute( - ast.Attribute(ast.Name("x", ast.Store()), "y", ast.Load()), - "z", - ast.Load(), + ast.Attribute( + ast.Name('x', ast.Store()), + 'y', ast.Load() + ), + 'z', ast.Load() ) ), ast.MatchValue( ast.Attribute( - ast.Attribute(ast.Name("x", ast.Load()), "y", ast.Store()), - "z", - ast.Load(), + ast.Attribute( + ast.Name('x', ast.Load()), + 'y', ast.Store() + ), + 'z', ast.Load() ) ), - ast.MatchValue(ast.Constant(...)), - ast.MatchValue(ast.Constant(True)), - ast.MatchValue(ast.Constant((1, 2, 3))), - ast.MatchSingleton("string"), - ast.MatchSequence([ast.MatchSingleton("string")]), - ast.MatchSequence([ast.MatchSequence([ast.MatchSingleton("string")])]), - ast.MatchMapping([constant_1, constant_true], [pattern_x]), + ast.MatchValue( + ast.Constant(...) + ), + ast.MatchValue( + ast.Constant(True) + ), + ast.MatchValue( + ast.Constant((1,2,3)) + ), + ast.MatchSingleton('string'), + ast.MatchSequence([ + ast.MatchSingleton('string') + ]), + ast.MatchSequence( + [ + ast.MatchSequence( + [ + ast.MatchSingleton('string') + ] + ) + ] + ), ast.MatchMapping( - [constant_true, constant_1], [pattern_x, pattern_1], rest="True" + [constant_1, constant_true], + [pattern_x] + ), + ast.MatchMapping( + [constant_true, constant_1], + [pattern_x, pattern_1], + rest='True' ), ast.MatchMapping( - [constant_true, ast.Starred(ast.Name("lol", ast.Load()), ast.Load())], + [constant_true, ast.Starred(ast.Name('lol', ast.Load()), ast.Load())], [pattern_x, pattern_1], - rest="legit", + rest='legit' ), ast.MatchClass( - ast.Attribute(ast.Attribute(constant_x, "y", ast.Load()), "z", ast.Load()), - patterns=[], - kwd_attrs=[], - kwd_patterns=[], + ast.Attribute( + ast.Attribute( + constant_x, + 'y', ast.Load()), + 'z', ast.Load()), + patterns=[], kwd_attrs=[], kwd_patterns=[] ), ast.MatchClass( - name_carter, patterns=[], kwd_attrs=["True"], kwd_patterns=[pattern_1] + name_carter, + patterns=[], + kwd_attrs=['True'], + kwd_patterns=[pattern_1] ), ast.MatchClass( - name_carter, patterns=[], kwd_attrs=[], kwd_patterns=[pattern_1] + name_carter, + patterns=[], + kwd_attrs=[], + kwd_patterns=[pattern_1] ), ast.MatchClass( name_carter, - patterns=[ast.MatchSingleton("string")], + patterns=[ast.MatchSingleton('string')], kwd_attrs=[], - kwd_patterns=[], + kwd_patterns=[] ), ast.MatchClass( - name_carter, patterns=[ast.MatchStar()], kwd_attrs=[], kwd_patterns=[] + name_carter, + patterns=[ast.MatchStar()], + kwd_attrs=[], + kwd_patterns=[] ), ast.MatchClass( - name_carter, patterns=[], kwd_attrs=[], kwd_patterns=[ast.MatchStar()] + name_carter, + patterns=[], + kwd_attrs=[], + kwd_patterns=[ast.MatchStar()] ), ast.MatchClass( constant_true, # invalid name patterns=[], - kwd_attrs=["True"], - kwd_patterns=[pattern_1], + kwd_attrs=['True'], + kwd_patterns=[pattern_1] + ), + ast.MatchSequence( + [ + ast.MatchStar("True") + ] + ), + ast.MatchAs( + name='False' + ), + ast.MatchOr( + [] + ), + ast.MatchOr( + [pattern_1] + ), + ast.MatchOr( + [pattern_1, pattern_x, ast.MatchSingleton('xxx')] ), - ast.MatchSequence([ast.MatchStar("True")]), - ast.MatchAs(name="False"), - ast.MatchOr([]), - ast.MatchOr([pattern_1]), - ast.MatchOr([pattern_1, pattern_x, ast.MatchSingleton("xxx")]), ast.MatchAs(name="_"), ast.MatchStar(name="x"), ast.MatchSequence([ast.MatchStar("_")]), ast.MatchMapping([], [], rest="_"), ] - @unittest.expectedFailure # TODO: RUSTPYTHON def test_match_validation_pattern(self): - name_x = ast.Name("x", ast.Load()) + name_x = ast.Name('x', ast.Load()) for pattern in self._MATCH_PATTERNS: with self.subTest(ast.dump(pattern, indent=4)): node = ast.Match( subject=name_x, - cases=[ast.match_case(pattern=pattern, body=[ast.Pass()])], + cases = [ + ast.match_case( + pattern=pattern, + body = [ast.Pass()] + ) + ] ) node = ast.fix_missing_locations(node) module = ast.Module([node], []) @@ -2426,44 +2615,35 @@ def compile_constant(self, value): ns = {} exec(code, ns) - return ns["x"] + return ns['x'] def test_validation(self): with self.assertRaises(TypeError) as cm: self.compile_constant([1, 2, 3]) - self.assertEqual(str(cm.exception), "got an invalid type in Constant: list") + self.assertEqual(str(cm.exception), + "got an invalid type in Constant: list") - @unittest.expectedFailure # TODO: RUSTPYTHON; b'' is not b'' def test_singletons(self): - for const in (None, False, True, Ellipsis, b"", frozenset()): + for const in (None, False, True, Ellipsis, b''): with self.subTest(const=const): value = self.compile_constant(const) self.assertIs(value, const) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_values(self): nested_tuple = (1,) nested_frozenset = frozenset({1}) for level in range(3): nested_tuple = (nested_tuple, 2) nested_frozenset = frozenset({nested_frozenset, 2}) - values = ( - 123, - 123.0, - 123j, - "unicode", - b"bytes", - tuple("tuple"), - frozenset("frozenset"), - nested_tuple, - nested_frozenset, - ) + values = (123, 123.0, 123j, + "unicode", b'bytes', + tuple("tuple"), frozenset("frozenset"), + nested_tuple, nested_frozenset) for value in values: with self.subTest(value=value): result = self.compile_constant(value) self.assertEqual(result, value) - @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError: cannot assign to literal def test_assign_to_constant(self): tree = ast.parse("x = 1") @@ -2474,35 +2654,42 @@ def test_assign_to_constant(self): with self.assertRaises(ValueError) as cm: compile(tree, "string", "exec") - self.assertEqual( - str(cm.exception), - "expression which can't be assigned " "to in Store context", - ) + self.assertEqual(str(cm.exception), + "expression which can't be assigned " + "to in Store context") def test_get_docstring(self): tree = ast.parse("'docstring'\nx = 1") - self.assertEqual(ast.get_docstring(tree), "docstring") + self.assertEqual(ast.get_docstring(tree), 'docstring') def get_load_const(self, tree): # Compile to bytecode, disassemble and get parameter of LOAD_CONST # instructions - co = compile(tree, "", "exec") + co = compile(tree, '', 'exec') consts = [] for instr in dis.get_instructions(co): - if instr.opname == "LOAD_CONST" or instr.opname == "RETURN_CONST": + if instr.opcode in dis.hasconst: consts.append(instr.argval) return consts @support.cpython_only def test_load_const(self): - consts = [None, True, False, 124, 2.0, 3j, "unicode", b"bytes", (1, 2, 3)] - - code = "\n".join(["x={!r}".format(const) for const in consts]) - code += "\nx = ..." + consts = [None, + True, False, + 1000, + 2.0, + 3j, + "unicode", + b'bytes', + (1, 2, 3)] + + code = '\n'.join(['x={!r}'.format(const) for const in consts]) + code += '\nx = ...' consts.extend((Ellipsis, None)) tree = ast.parse(code) - self.assertEqual(self.get_load_const(tree), consts) + self.assertEqual(self.get_load_const(tree), + consts) # Replace expression nodes with constants for assign, const in zip(tree.body, consts): @@ -2511,7 +2698,8 @@ def test_load_const(self): ast.copy_location(new_node, assign.value) assign.value = new_node - self.assertEqual(self.get_load_const(tree), consts) + self.assertEqual(self.get_load_const(tree), + consts) def test_literal_eval(self): tree = ast.parse("1 + 2") @@ -2525,22 +2713,22 @@ def test_literal_eval(self): ast.copy_location(new_right, binop.right) binop.right = new_right - self.assertEqual(ast.literal_eval(binop), 10 + 20j) + self.assertEqual(ast.literal_eval(binop), 10+20j) def test_string_kind(self): - c = ast.parse('"x"', mode="eval").body + c = ast.parse('"x"', mode='eval').body self.assertEqual(c.value, "x") self.assertEqual(c.kind, None) - c = ast.parse('u"x"', mode="eval").body + c = ast.parse('u"x"', mode='eval').body self.assertEqual(c.value, "x") self.assertEqual(c.kind, "u") - c = ast.parse('r"x"', mode="eval").body + c = ast.parse('r"x"', mode='eval').body self.assertEqual(c.value, "x") self.assertEqual(c.kind, None) - c = ast.parse('b"x"', mode="eval").body + c = ast.parse('b"x"', mode='eval').body self.assertEqual(c.value, b"x") self.assertEqual(c.kind, None) @@ -2551,7 +2739,6 @@ class EndPositionTests(unittest.TestCase): Testing end positions of nodes requires a bit of extra care because of how LL parsers work. """ - def _check_end_pos(self, ast_node, end_lineno, end_col_offset): self.assertEqual(ast_node.end_lineno, end_lineno) self.assertEqual(ast_node.end_col_offset, end_col_offset) @@ -2565,55 +2752,55 @@ def _parse_value(self, s): return ast.parse(s).body[0].value def test_lambda(self): - s = "lambda x, *y: None" + s = 'lambda x, *y: None' lam = self._parse_value(s) - self._check_content(s, lam.body, "None") - self._check_content(s, lam.args.args[0], "x") - self._check_content(s, lam.args.vararg, "y") + self._check_content(s, lam.body, 'None') + self._check_content(s, lam.args.args[0], 'x') + self._check_content(s, lam.args.vararg, 'y') def test_func_def(self): - s = dedent(""" + s = dedent(''' def func(x: int, *args: str, z: float = 0, **kwargs: Any) -> bool: return True - """).strip() + ''').strip() fdef = ast.parse(s).body[0] self._check_end_pos(fdef, 5, 15) - self._check_content(s, fdef.body[0], "return True") - self._check_content(s, fdef.args.args[0], "x: int") - self._check_content(s, fdef.args.args[0].annotation, "int") - self._check_content(s, fdef.args.kwarg, "kwargs: Any") - self._check_content(s, fdef.args.kwarg.annotation, "Any") + self._check_content(s, fdef.body[0], 'return True') + self._check_content(s, fdef.args.args[0], 'x: int') + self._check_content(s, fdef.args.args[0].annotation, 'int') + self._check_content(s, fdef.args.kwarg, 'kwargs: Any') + self._check_content(s, fdef.args.kwarg.annotation, 'Any') def test_call(self): - s = "func(x, y=2, **kw)" + s = 'func(x, y=2, **kw)' call = self._parse_value(s) - self._check_content(s, call.func, "func") - self._check_content(s, call.keywords[0].value, "2") - self._check_content(s, call.keywords[1].value, "kw") + self._check_content(s, call.func, 'func') + self._check_content(s, call.keywords[0].value, '2') + self._check_content(s, call.keywords[1].value, 'kw') def test_call_noargs(self): - s = "x[0]()" + s = 'x[0]()' call = self._parse_value(s) - self._check_content(s, call.func, "x[0]") + self._check_content(s, call.func, 'x[0]') self._check_end_pos(call, 1, 6) def test_class_def(self): - s = dedent(""" + s = dedent(''' class C(A, B): x: int = 0 - """).strip() + ''').strip() cdef = ast.parse(s).body[0] self._check_end_pos(cdef, 2, 14) - self._check_content(s, cdef.bases[1], "B") - self._check_content(s, cdef.body[0], "x: int = 0") + self._check_content(s, cdef.bases[1], 'B') + self._check_content(s, cdef.body[0], 'x: int = 0') def test_class_kw(self): - s = "class S(metaclass=abc.ABCMeta): pass" + s = 'class S(metaclass=abc.ABCMeta): pass' cdef = ast.parse(s).body[0] - self._check_content(s, cdef.keywords[0].value, "abc.ABCMeta") + self._check_content(s, cdef.keywords[0].value, 'abc.ABCMeta') def test_multi_line_str(self): s = dedent(''' @@ -2626,10 +2813,10 @@ def test_multi_line_str(self): self._check_end_pos(assign.value, 3, 40) def test_continued_str(self): - s = dedent(""" + s = dedent(''' x = "first part" \\ "second part" - """).strip() + ''').strip() assign = ast.parse(s).body[0] self._check_end_pos(assign, 2, 13) self._check_end_pos(assign.value, 2, 13) @@ -2637,7 +2824,7 @@ def test_continued_str(self): def test_suites(self): # We intentionally put these into the same string to check # that empty lines are not part of the suite. - s = dedent(""" + s = dedent(''' while True: pass @@ -2657,7 +2844,7 @@ def test_suites(self): pass pass - """).strip() + ''').strip() mod = ast.parse(s) while_loop = mod.body[0] if_stmt = mod.body[1] @@ -2671,18 +2858,18 @@ def test_suites(self): self._check_end_pos(try_stmt, 17, 8) self._check_end_pos(pass_stmt, 19, 4) - self._check_content(s, while_loop.test, "True") - self._check_content(s, if_stmt.body[0], "x = None") - self._check_content(s, if_stmt.orelse[0].test, "other()") - self._check_content(s, for_loop.target, "x, y") - self._check_content(s, try_stmt.body[0], "raise RuntimeError") - self._check_content(s, try_stmt.handlers[0].type, "TypeError") + self._check_content(s, while_loop.test, 'True') + self._check_content(s, if_stmt.body[0], 'x = None') + self._check_content(s, if_stmt.orelse[0].test, 'other()') + self._check_content(s, for_loop.target, 'x, y') + self._check_content(s, try_stmt.body[0], 'raise RuntimeError') + self._check_content(s, try_stmt.handlers[0].type, 'TypeError') def test_fstring(self): s = 'x = f"abc {x + y} abc"' fstr = self._parse_value(s) binop = fstr.values[1].value - self._check_content(s, binop, "x + y") + self._check_content(s, binop, 'x + y') def test_fstring_multi_line(self): s = dedent(''' @@ -2697,198 +2884,200 @@ def test_fstring_multi_line(self): fstr = self._parse_value(s) binop = fstr.values[1].value self._check_end_pos(binop, 5, 7) - self._check_content(s, binop.left, "arg_one") - self._check_content(s, binop.right, "arg_two") + self._check_content(s, binop.left, 'arg_one') + self._check_content(s, binop.right, 'arg_two') def test_import_from_multi_line(self): - s = dedent(""" + s = dedent(''' from x.y.z import ( a, b, c as c ) - """).strip() + ''').strip() imp = ast.parse(s).body[0] self._check_end_pos(imp, 3, 1) self._check_end_pos(imp.names[2], 2, 16) def test_slices(self): - s1 = "f()[1, 2] [0]" - s2 = "x[ a.b: c.d]" - sm = dedent(""" + s1 = 'f()[1, 2] [0]' + s2 = 'x[ a.b: c.d]' + sm = dedent(''' x[ a.b: f () , g () : c.d ] - """).strip() + ''').strip() i1, i2, im = map(self._parse_value, (s1, s2, sm)) - self._check_content(s1, i1.value, "f()[1, 2]") - self._check_content(s1, i1.value.slice, "1, 2") - self._check_content(s2, i2.slice.lower, "a.b") - self._check_content(s2, i2.slice.upper, "c.d") - self._check_content(sm, im.slice.elts[0].upper, "f ()") - self._check_content(sm, im.slice.elts[1].lower, "g ()") + self._check_content(s1, i1.value, 'f()[1, 2]') + self._check_content(s1, i1.value.slice, '1, 2') + self._check_content(s2, i2.slice.lower, 'a.b') + self._check_content(s2, i2.slice.upper, 'c.d') + self._check_content(sm, im.slice.elts[0].upper, 'f ()') + self._check_content(sm, im.slice.elts[1].lower, 'g ()') self._check_end_pos(im, 3, 3) def test_binop(self): - s = dedent(""" + s = dedent(''' (1 * 2 + (3 ) + 4 ) - """).strip() + ''').strip() binop = self._parse_value(s) self._check_end_pos(binop, 2, 6) - self._check_content(s, binop.right, "4") - self._check_content(s, binop.left, "1 * 2 + (3 )") - self._check_content(s, binop.left.right, "3") + self._check_content(s, binop.right, '4') + self._check_content(s, binop.left, '1 * 2 + (3 )') + self._check_content(s, binop.left.right, '3') def test_boolop(self): - s = dedent(""" + s = dedent(''' if (one_condition and (other_condition or yet_another_one)): pass - """).strip() + ''').strip() bop = ast.parse(s).body[0].test self._check_end_pos(bop, 2, 44) - self._check_content(s, bop.values[1], "other_condition or yet_another_one") + self._check_content(s, bop.values[1], + 'other_condition or yet_another_one') def test_tuples(self): - s1 = "x = () ;" - s2 = "x = 1 , ;" - s3 = "x = (1 , 2 ) ;" - sm = dedent(""" + s1 = 'x = () ;' + s2 = 'x = 1 , ;' + s3 = 'x = (1 , 2 ) ;' + sm = dedent(''' x = ( a, b, ) - """).strip() + ''').strip() t1, t2, t3, tm = map(self._parse_value, (s1, s2, s3, sm)) - self._check_content(s1, t1, "()") - self._check_content(s2, t2, "1 ,") - self._check_content(s3, t3, "(1 , 2 )") + self._check_content(s1, t1, '()') + self._check_content(s2, t2, '1 ,') + self._check_content(s3, t3, '(1 , 2 )') self._check_end_pos(tm, 3, 1) def test_attribute_spaces(self): - s = "func(x. y .z)" + s = 'func(x. y .z)' call = self._parse_value(s) self._check_content(s, call, s) - self._check_content(s, call.args[0], "x. y .z") + self._check_content(s, call.args[0], 'x. y .z') def test_redundant_parenthesis(self): - s = "( ( ( a + b ) ) )" + s = '( ( ( a + b ) ) )' v = ast.parse(s).body[0].value - self.assertEqual(type(v).__name__, "BinOp") - self._check_content(s, v, "a + b") - s2 = "await " + s + self.assertEqual(type(v).__name__, 'BinOp') + self._check_content(s, v, 'a + b') + s2 = 'await ' + s v = ast.parse(s2).body[0].value.value - self.assertEqual(type(v).__name__, "BinOp") - self._check_content(s2, v, "a + b") + self.assertEqual(type(v).__name__, 'BinOp') + self._check_content(s2, v, 'a + b') def test_trailers_with_redundant_parenthesis(self): tests = ( - ("( ( ( a ) ) ) ( )", "Call"), - ("( ( ( a ) ) ) ( b )", "Call"), - ("( ( ( a ) ) ) [ b ]", "Subscript"), - ("( ( ( a ) ) ) . b", "Attribute"), + ('( ( ( a ) ) ) ( )', 'Call'), + ('( ( ( a ) ) ) ( b )', 'Call'), + ('( ( ( a ) ) ) [ b ]', 'Subscript'), + ('( ( ( a ) ) ) . b', 'Attribute'), ) for s, t in tests: with self.subTest(s): v = ast.parse(s).body[0].value self.assertEqual(type(v).__name__, t) self._check_content(s, v, s) - s2 = "await " + s + s2 = 'await ' + s v = ast.parse(s2).body[0].value.value self.assertEqual(type(v).__name__, t) self._check_content(s2, v, s) def test_displays(self): - s1 = "[{}, {1, }, {1, 2,} ]" - s2 = "{a: b, f (): g () ,}" + s1 = '[{}, {1, }, {1, 2,} ]' + s2 = '{a: b, f (): g () ,}' c1 = self._parse_value(s1) c2 = self._parse_value(s2) - self._check_content(s1, c1.elts[0], "{}") - self._check_content(s1, c1.elts[1], "{1, }") - self._check_content(s1, c1.elts[2], "{1, 2,}") - self._check_content(s2, c2.keys[1], "f ()") - self._check_content(s2, c2.values[1], "g ()") + self._check_content(s1, c1.elts[0], '{}') + self._check_content(s1, c1.elts[1], '{1, }') + self._check_content(s1, c1.elts[2], '{1, 2,}') + self._check_content(s2, c2.keys[1], 'f ()') + self._check_content(s2, c2.values[1], 'g ()') def test_comprehensions(self): - s = dedent(""" + s = dedent(''' x = [{x for x, y in stuff if cond.x} for stuff in things] - """).strip() + ''').strip() cmp = self._parse_value(s) self._check_end_pos(cmp, 2, 37) - self._check_content(s, cmp.generators[0].iter, "things") - self._check_content(s, cmp.elt.generators[0].iter, "stuff") - self._check_content(s, cmp.elt.generators[0].ifs[0], "cond.x") - self._check_content(s, cmp.elt.generators[0].target, "x, y") + self._check_content(s, cmp.generators[0].iter, 'things') + self._check_content(s, cmp.elt.generators[0].iter, 'stuff') + self._check_content(s, cmp.elt.generators[0].ifs[0], 'cond.x') + self._check_content(s, cmp.elt.generators[0].target, 'x, y') def test_yield_await(self): - s = dedent(""" + s = dedent(''' async def f(): yield x await y - """).strip() + ''').strip() fdef = ast.parse(s).body[0] - self._check_content(s, fdef.body[0].value, "yield x") - self._check_content(s, fdef.body[1].value, "await y") + self._check_content(s, fdef.body[0].value, 'yield x') + self._check_content(s, fdef.body[1].value, 'await y') def test_source_segment_multi(self): - s_orig = dedent(""" + s_orig = dedent(''' x = ( a, b, ) + () - """).strip() - s_tuple = dedent(""" + ''').strip() + s_tuple = dedent(''' ( a, b, ) - """).strip() + ''').strip() binop = self._parse_value(s_orig) self.assertEqual(ast.get_source_segment(s_orig, binop.left), s_tuple) def test_source_segment_padded(self): - s_orig = dedent(""" + s_orig = dedent(''' class C: def fun(self) -> None: "ЖЖЖЖЖ" - """).strip() - s_method = " def fun(self) -> None:\n" ' "ЖЖЖЖЖ"' + ''').strip() + s_method = ' def fun(self) -> None:\n' \ + ' "ЖЖЖЖЖ"' cdef = ast.parse(s_orig).body[0] - self.assertEqual( - ast.get_source_segment(s_orig, cdef.body[0], padded=True), s_method - ) + self.assertEqual(ast.get_source_segment(s_orig, cdef.body[0], padded=True), + s_method) def test_source_segment_endings(self): - s = "v = 1\r\nw = 1\nx = 1\n\ry = 1\rz = 1\r\n" + s = 'v = 1\r\nw = 1\nx = 1\n\ry = 1\rz = 1\r\n' v, w, x, y, z = ast.parse(s).body - self._check_content(s, v, "v = 1") - self._check_content(s, w, "w = 1") - self._check_content(s, x, "x = 1") - self._check_content(s, y, "y = 1") - self._check_content(s, z, "z = 1") + self._check_content(s, v, 'v = 1') + self._check_content(s, w, 'w = 1') + self._check_content(s, x, 'x = 1') + self._check_content(s, y, 'y = 1') + self._check_content(s, z, 'z = 1') def test_source_segment_tabs(self): - s = dedent(""" + s = dedent(''' class C: \t\f def fun(self) -> None: \t\f pass - """).strip() - s_method = " \t\f def fun(self) -> None:\n" " \t\f pass" + ''').strip() + s_method = ' \t\f def fun(self) -> None:\n' \ + ' \t\f pass' cdef = ast.parse(s).body[0] self.assertEqual(ast.get_source_segment(s, cdef.body[0], padded=True), s_method) def test_source_segment_newlines(self): - s = "def f():\n pass\ndef g():\r pass\r\ndef h():\r\n pass\r\n" + s = 'def f():\n pass\ndef g():\r pass\r\ndef h():\r\n pass\r\n' f, g, h = ast.parse(s).body - self._check_content(s, f, "def f():\n pass") - self._check_content(s, g, "def g():\r pass") - self._check_content(s, h, "def h():\r\n pass") + self._check_content(s, f, 'def f():\n pass') + self._check_content(s, g, 'def g():\r pass') + self._check_content(s, h, 'def h():\r\n pass') - s = "def f():\n a = 1\r b = 2\r\n c = 3\n" + s = 'def f():\n a = 1\r b = 2\r\n c = 3\n' f = ast.parse(s).body[0] self._check_content(s, f, s.rstrip()) def test_source_segment_missing_info(self): - s = "v = 1\r\nw = 1\nx = 1\n\ry = 1\r\n" + s = 'v = 1\r\nw = 1\nx = 1\n\ry = 1\r\n' v, w, x, y = ast.parse(s).body del v.lineno del w.end_lineno @@ -2900,102 +3089,27 @@ def test_source_segment_missing_info(self): self.assertIsNone(ast.get_source_segment(s, y)) -class BaseNodeVisitorCases: - # Both `NodeVisitor` and `NodeTranformer` must raise these warnings: - def test_old_constant_nodes(self): - class Visitor(self.visitor_class): - def visit_Num(self, node): - log.append((node.lineno, "Num", node.n)) - - def visit_Str(self, node): - log.append((node.lineno, "Str", node.s)) - - def visit_Bytes(self, node): - log.append((node.lineno, "Bytes", node.s)) - - def visit_NameConstant(self, node): - log.append((node.lineno, "NameConstant", node.value)) - - def visit_Ellipsis(self, node): - log.append((node.lineno, "Ellipsis", ...)) - - mod = ast.parse( - dedent("""\ - i = 42 - f = 4.25 - c = 4.25j - s = 'string' - b = b'bytes' - t = True - n = None - e = ... - """) - ) - visitor = Visitor() - log = [] - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings("always", "", DeprecationWarning) - visitor.visit(mod) - self.assertEqual( - log, - [ - (1, "Num", 42), - (2, "Num", 4.25), - (3, "Num", 4.25j), - (4, "Str", "string"), - (5, "Bytes", b"bytes"), - (6, "NameConstant", True), - (7, "NameConstant", None), - (8, "Ellipsis", ...), - ], - ) - self.assertEqual( - [str(w.message) for w in wlog], - [ - "visit_Num is deprecated; add visit_Constant", - "Attribute n is deprecated and will be removed in Python 3.14; use value instead", - "visit_Num is deprecated; add visit_Constant", - "Attribute n is deprecated and will be removed in Python 3.14; use value instead", - "visit_Num is deprecated; add visit_Constant", - "Attribute n is deprecated and will be removed in Python 3.14; use value instead", - "visit_Str is deprecated; add visit_Constant", - "Attribute s is deprecated and will be removed in Python 3.14; use value instead", - "visit_Bytes is deprecated; add visit_Constant", - "Attribute s is deprecated and will be removed in Python 3.14; use value instead", - "visit_NameConstant is deprecated; add visit_Constant", - "visit_NameConstant is deprecated; add visit_Constant", - "visit_Ellipsis is deprecated; add visit_Constant", - ], - ) - - -class NodeVisitorTests(BaseNodeVisitorCases, unittest.TestCase): - visitor_class = ast.NodeVisitor - - -class NodeTransformerTests(ASTTestMixin, BaseNodeVisitorCases, unittest.TestCase): - visitor_class = ast.NodeTransformer - - def assertASTTransformation(self, tranformer_class, initial_code, expected_code): +class NodeTransformerTests(ASTTestMixin, unittest.TestCase): + def assertASTTransformation(self, transformer_class, + initial_code, expected_code): initial_ast = ast.parse(dedent(initial_code)) expected_ast = ast.parse(dedent(expected_code)) - tranformer = tranformer_class() - result_ast = ast.fix_missing_locations(tranformer.visit(initial_ast)) + transformer = transformer_class() + result_ast = ast.fix_missing_locations(transformer.visit(initial_ast)) self.assertASTEqual(result_ast, expected_ast) - @unittest.expectedFailure # TODO: RUSTPYTHON; is not def test_node_remove_single(self): - code = "def func(arg) -> SomeType: ..." - expected = "def func(arg): ..." + code = 'def func(arg) -> SomeType: ...' + expected = 'def func(arg): ...' # Since `FunctionDef.returns` is defined as a single value, we test # the `if isinstance(old_value, AST):` branch here. class SomeTypeRemover(ast.NodeTransformer): def visit_Name(self, node: ast.Name): self.generic_visit(node) - if node.id == "SomeType": + if node.id == 'SomeType': return None return node @@ -3034,11 +3148,11 @@ class DSL(Base, kw1=True, kw2=True, kw3=False): ... class ExtendKeywords(ast.NodeTransformer): def visit_keyword(self, node: ast.keyword): self.generic_visit(node) - if node.arg == "kw1": + if node.arg == 'kw1': return [ node, - ast.keyword("kw2", ast.Constant(True)), - ast.keyword("kw3", ast.Constant(False)), + ast.keyword('kw2', ast.Constant(True)), + ast.keyword('kw3', ast.Constant(False)), ] return node @@ -3057,8 +3171,8 @@ def func(arg): class PrintToLog(ast.NodeTransformer): def visit_Call(self, node: ast.Call): self.generic_visit(node) - if isinstance(node.func, ast.Name) and node.func.id == "print": - node.func.id = "log" + if isinstance(node.func, ast.Name) and node.func.id == 'print': + node.func.id = 'log' return node self.assertASTTransformation(PrintToLog, code, expected) @@ -3076,15 +3190,15 @@ def func(arg): class PrintToLog(ast.NodeTransformer): def visit_Call(self, node: ast.Call): self.generic_visit(node) - if isinstance(node.func, ast.Name) and node.func.id == "print": + if isinstance(node.func, ast.Name) and node.func.id == 'print': return ast.Call( func=ast.Attribute( - ast.Name("logger", ctx=ast.Load()), - attr="log", + ast.Name('logger', ctx=ast.Load()), + attr='log', ctx=ast.Load(), ), args=node.args, - keywords=[ast.keyword("debug", ast.Constant(True))], + keywords=[ast.keyword('debug', ast.Constant(True))], ) return node @@ -3094,23 +3208,19 @@ def visit_Call(self, node: ast.Call): class ASTConstructorTests(unittest.TestCase): """Test the autogenerated constructors for AST nodes.""" - @unittest.expectedFailure # TODO: RUSTPYTHON def test_FunctionDef(self): args = ast.arguments() self.assertEqual(args.args, []) self.assertEqual(args.posonlyargs, []) - with self.assertWarnsRegex( - DeprecationWarning, - r"FunctionDef\.__init__ missing 1 required positional argument: 'name'", - ): + with self.assertWarnsRegex(DeprecationWarning, + r"FunctionDef\.__init__ missing 1 required positional argument: 'name'"): node = ast.FunctionDef(args=args) - self.assertFalse(hasattr(node, "name")) + self.assertNotHasAttr(node, "name") self.assertEqual(node.decorator_list, []) - node = ast.FunctionDef(name="foo", args=args) - self.assertEqual(node.name, "foo") + node = ast.FunctionDef(name='foo', args=args) + self.assertEqual(node.name, 'foo') self.assertEqual(node.decorator_list, []) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_expr_context(self): name = ast.Name("x") self.assertEqual(name.id, "x") @@ -3124,10 +3234,8 @@ def test_expr_context(self): self.assertEqual(name3.id, "x") self.assertIsInstance(name3.ctx, ast.Del) - with self.assertWarnsRegex( - DeprecationWarning, - r"Name\.__init__ missing 1 required positional argument: 'id'", - ): + with self.assertWarnsRegex(DeprecationWarning, + r"Name\.__init__ missing 1 required positional argument: 'id'"): name3 = ast.Name() def test_custom_subclass_with_no_fields(self): @@ -3140,7 +3248,7 @@ class NoInit(ast.AST): def test_fields_but_no_field_types(self): class Fields(ast.AST): - _fields = ("a",) + _fields = ('a',) obj = Fields() with self.assertRaises(AttributeError): @@ -3150,8 +3258,8 @@ class Fields(ast.AST): def test_fields_and_types(self): class FieldsAndTypes(ast.AST): - _fields = ("a",) - _field_types = {"a": int | None} + _fields = ('a',) + _field_types = {'a': int | None} a: int | None = None obj = FieldsAndTypes() @@ -3159,7 +3267,6 @@ class FieldsAndTypes(ast.AST): obj = FieldsAndTypes(a=1) self.assertEqual(obj.a, 1) - @unittest.expectedFailure # TODO: RUSTPYTHON; DeprecationWarning not triggered def test_custom_attributes(self): class MyAttrs(ast.AST): _attributes = ("a", "b") @@ -3168,39 +3275,35 @@ class MyAttrs(ast.AST): self.assertEqual(obj.a, 1) self.assertEqual(obj.b, 2) - with self.assertWarnsRegex( - DeprecationWarning, - r"MyAttrs.__init__ got an unexpected keyword argument 'c'.", - ): + with self.assertWarnsRegex(DeprecationWarning, + r"MyAttrs.__init__ got an unexpected keyword argument 'c'."): obj = MyAttrs(c=3) - @unittest.expectedFailure # TODO: RUSTPYTHON; DeprecationWarning not triggered + @unittest.expectedFailure # TODO: RUSTPYTHON; DeprecationWarning not triggered def test_fields_and_types_no_default(self): class FieldsAndTypesNoDefault(ast.AST): - _fields = ("a",) - _field_types = {"a": int} + _fields = ('a',) + _field_types = {'a': int} - with self.assertWarnsRegex( - DeprecationWarning, - r"FieldsAndTypesNoDefault\.__init__ missing 1 required positional argument: 'a'\.", - ): + with self.assertWarnsRegex(DeprecationWarning, + r"FieldsAndTypesNoDefault\.__init__ missing 1 required positional argument: 'a'\."): obj = FieldsAndTypesNoDefault() with self.assertRaises(AttributeError): obj.a obj = FieldsAndTypesNoDefault(a=1) self.assertEqual(obj.a, 1) - @unittest.expectedFailure # TODO: RUSTPYTHON; DeprecationWarning not triggered + @unittest.expectedFailure # TODO: RUSTPYTHON; DeprecationWarning not triggered def test_incomplete_field_types(self): class MoreFieldsThanTypes(ast.AST): - _fields = ("a", "b") - _field_types = {"a": int | None} + _fields = ('a', 'b') + _field_types = {'a': int | None} a: int | None = None b: int | None = None with self.assertWarnsRegex( DeprecationWarning, - r"Field 'b' is missing from MoreFieldsThanTypes\._field_types", + r"Field 'b' is missing from MoreFieldsThanTypes\._field_types" ): obj = MoreFieldsThanTypes() self.assertIs(obj.a, None) @@ -3210,11 +3313,20 @@ class MoreFieldsThanTypes(ast.AST): self.assertEqual(obj.a, 1) self.assertEqual(obj.b, 2) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Expected type 'str' but 'bytes' found. + def test_malformed_fields_with_bytes(self): + class BadFields(ast.AST): + _fields = (b'\xff'*64,) + _field_types = {'a': int} + + # This should not crash + with self.assertWarnsRegex(DeprecationWarning, r"Field b'\\xff\\xff.*' .*"): + obj = BadFields() + def test_complete_field_types(self): class _AllFieldTypes(ast.AST): - _fields = ("a", "b") - _field_types = {"a": int | None, "b": list[str]} + _fields = ('a', 'b') + _field_types = {'a': int | None, 'b': list[str]} # This must be set explicitly a: int | None = None # This will add an implicit empty list default @@ -3224,6 +3336,28 @@ class _AllFieldTypes(ast.AST): self.assertIs(obj.a, None) self.assertEqual(obj.b, []) + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_non_str_kwarg(self): + warn_msg = "got an unexpected keyword argument int: + x -= 1 + return x + ''') + + for r in range(1, len(base_flags) + 1): + for choices in itertools.combinations(base_flags, r=r): + for args in itertools.product(*choices): + with self.subTest(flags=args): + self.invoke_ast(*args) + + @support.force_not_colorized + def test_help_message(self): + for flag in ('-h', '--help', '--unknown'): + with self.subTest(flag=flag): + output = StringIO() + with self.assertRaises(SystemExit): + with contextlib.redirect_stderr(output): + ast.main(args=flag) + self.assertStartsWith(output.getvalue(), 'usage: ') + + def test_exec_mode_flag(self): + # test 'python -m ast -m/--mode exec' + source = 'x: bool = 1 # type: ignore[assignment]' + expect = ''' + Module( + body=[ + AnnAssign( + target=Name(id='x', ctx=Store()), + annotation=Name(id='bool', ctx=Load()), + value=Constant(value=1), + simple=1)], + type_ignores=[ + TypeIgnore(lineno=1, tag='[assignment]')]) + ''' + for flag in ('-m=exec', '--mode=exec'): + with self.subTest(flag=flag): + self.check_output(source, expect, flag) + + def test_single_mode_flag(self): + # test 'python -m ast -m/--mode single' + source = 'pass' + expect = ''' + Interactive( + body=[ + Pass()]) + ''' + for flag in ('-m=single', '--mode=single'): + with self.subTest(flag=flag): + self.check_output(source, expect, flag) + + def test_eval_mode_flag(self): + # test 'python -m ast -m/--mode eval' + source = 'print(1, 2, 3)' + expect = ''' + Expression( + body=Call( + func=Name(id='print', ctx=Load()), + args=[ + Constant(value=1), + Constant(value=2), + Constant(value=3)])) + ''' + for flag in ('-m=eval', '--mode=eval'): + with self.subTest(flag=flag): + self.check_output(source, expect, flag) + + def test_func_type_mode_flag(self): + # test 'python -m ast -m/--mode func_type' + source = '(int, str) -> list[int]' + expect = ''' + FunctionType( + argtypes=[ + Name(id='int', ctx=Load()), + Name(id='str', ctx=Load())], + returns=Subscript( + value=Name(id='list', ctx=Load()), + slice=Name(id='int', ctx=Load()), + ctx=Load())) + ''' + for flag in ('-m=func_type', '--mode=func_type'): + with self.subTest(flag=flag): + self.check_output(source, expect, flag) + + def test_no_type_comments_flag(self): + # test 'python -m ast --no-type-comments' + source = 'x: bool = 1 # type: ignore[assignment]' + expect = ''' + Module( + body=[ + AnnAssign( + target=Name(id='x', ctx=Store()), + annotation=Name(id='bool', ctx=Load()), + value=Constant(value=1), + simple=1)]) + ''' + self.check_output(source, expect, '--no-type-comments') + + def test_include_attributes_flag(self): + # test 'python -m ast -a/--include-attributes' + source = 'pass' + expect = ''' + Module( + body=[ + Pass( + lineno=1, + col_offset=0, + end_lineno=1, + end_col_offset=4)]) + ''' + for flag in ('-a', '--include-attributes'): + with self.subTest(flag=flag): + self.check_output(source, expect, flag) + + def test_indent_flag(self): + # test 'python -m ast -i/--indent 0' + source = 'pass' + expect = ''' + Module( + body=[ + Pass()]) + ''' + for flag in ('-i=0', '--indent=0'): + with self.subTest(flag=flag): + self.check_output(source, expect, flag) + + def test_feature_version_flag(self): + # test 'python -m ast --feature-version 3.9/3.10' + source = ''' + match x: + case 1: + pass + ''' + expect = ''' + Module( + body=[ + Match( + subject=Name(id='x', ctx=Load()), + cases=[ + match_case( + pattern=MatchValue( + value=Constant(value=1)), + body=[ + Pass()])])]) + ''' + self.check_output(source, expect, '--feature-version=3.10') + with self.assertRaises(SyntaxError): + self.invoke_ast('--feature-version=3.9') + + def test_no_optimize_flag(self): + # test 'python -m ast -O/--optimize -1/0' + source = ''' + match a: + case 1+2j: + pass + ''' + expect = ''' + Module( + body=[ + Match( + subject=Name(id='a', ctx=Load()), + cases=[ + match_case( + pattern=MatchValue( + value=BinOp( + left=Constant(value=1), + op=Add(), + right=Constant(value=2j))), + body=[ + Pass()])])]) + ''' + for flag in ('-O=-1', '--optimize=-1', '-O=0', '--optimize=0'): + with self.subTest(flag=flag): + self.check_output(source, expect, flag) + + def test_optimize_flag(self): + # test 'python -m ast -O/--optimize 1/2' + source = ''' + match a: + case 1+2j: + pass + ''' + expect = ''' + Module( + body=[ + Match( + subject=Name(id='a', ctx=Load()), + cases=[ + match_case( + pattern=MatchValue( + value=Constant(value=(1+2j))), + body=[ + Pass()])])]) + ''' + for flag in ('-O=1', '--optimize=1', '-O=2', '--optimize=2'): + with self.subTest(flag=flag): + self.check_output(source, expect, flag) + + def test_show_empty_flag(self): + # test 'python -m ast --show-empty' + source = 'print(1, 2, 3)' + expect = ''' + Module( + body=[ + Expr( + value=Call( + func=Name(id='print', ctx=Load()), + args=[ + Constant(value=1), + Constant(value=2), + Constant(value=3)], + keywords=[]))], + type_ignores=[]) + ''' + self.check_output(source, expect, '--show-empty') -def compare(left, right): - return ast.dump(left) == ast.dump(right) class ASTOptimizationTests(unittest.TestCase): - binop = { - "+": ast.Add(), - "-": ast.Sub(), - "*": ast.Mult(), - "/": ast.Div(), - "%": ast.Mod(), - "<<": ast.LShift(), - ">>": ast.RShift(), - "|": ast.BitOr(), - "^": ast.BitXor(), - "&": ast.BitAnd(), - "//": ast.FloorDiv(), - "**": ast.Pow(), - } - - unaryop = { - "~": ast.Invert(), - "+": ast.UAdd(), - "-": ast.USub(), - } - def wrap_expr(self, expr): return ast.Module(body=[ast.Expr(value=expr)]) @@ -3358,112 +3709,31 @@ def wrap_statement(self, statement): return ast.Module(body=[statement]) def assert_ast(self, code, non_optimized_target, optimized_target): - non_optimized_tree = ast.parse(code, optimize=-1) optimized_tree = ast.parse(code, optimize=1) # Is a non-optimized tree equal to a non-optimized target? self.assertTrue( - compare(non_optimized_tree, non_optimized_target), + ast.compare(non_optimized_tree, non_optimized_target), f"{ast.dump(non_optimized_target)} must equal " f"{ast.dump(non_optimized_tree)}", ) # Is a optimized tree equal to a non-optimized target? self.assertFalse( - compare(optimized_tree, non_optimized_target), + ast.compare(optimized_tree, non_optimized_target), f"{ast.dump(non_optimized_target)} must not equal " f"{ast.dump(non_optimized_tree)}" ) # Is a optimized tree is equal to an optimized target? self.assertTrue( - compare(optimized_tree, optimized_target), + ast.compare(optimized_tree, optimized_target), f"{ast.dump(optimized_target)} must equal " f"{ast.dump(optimized_tree)}", ) - def create_binop(self, operand, left=ast.Constant(1), right=ast.Constant(1)): - return ast.BinOp(left=left, op=self.binop[operand], right=right) - - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags - def test_folding_binop(self): - code = "1 %s 1" - operators = self.binop.keys() - - for op in operators: - result_code = code % op - non_optimized_target = self.wrap_expr(self.create_binop(op)) - optimized_target = self.wrap_expr(ast.Constant(value=eval(result_code))) - - with self.subTest( - result_code=result_code, - non_optimized_target=non_optimized_target, - optimized_target=optimized_target - ): - self.assert_ast(result_code, non_optimized_target, optimized_target) - - # Multiplication of constant tuples must be folded - code = "(1,) * 3" - non_optimized_target = self.wrap_expr(self.create_binop("*", ast.Tuple(elts=[ast.Constant(value=1)]), ast.Constant(value=3))) - optimized_target = self.wrap_expr(ast.Constant(eval(code))) - - self.assert_ast(code, non_optimized_target, optimized_target) - - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags - def test_folding_unaryop(self): - code = "%s1" - operators = self.unaryop.keys() - - def create_unaryop(operand): - return ast.UnaryOp(op=self.unaryop[operand], operand=ast.Constant(1)) - - for op in operators: - result_code = code % op - non_optimized_target = self.wrap_expr(create_unaryop(op)) - optimized_target = self.wrap_expr(ast.Constant(eval(result_code))) - - with self.subTest( - result_code=result_code, - non_optimized_target=non_optimized_target, - optimized_target=optimized_target - ): - self.assert_ast(result_code, non_optimized_target, optimized_target) - - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags - def test_folding_not(self): - code = "not (1 %s (1,))" - operators = { - "in": ast.In(), - "is": ast.Is(), - } - opt_operators = { - "is": ast.IsNot(), - "in": ast.NotIn(), - } - - def create_notop(operand): - return ast.UnaryOp(op=ast.Not(), operand=ast.Compare( - left=ast.Constant(value=1), - ops=[operators[operand]], - comparators=[ast.Tuple(elts=[ast.Constant(value=1)])] - )) - - for op in operators.keys(): - result_code = code % op - non_optimized_target = self.wrap_expr(create_notop(op)) - optimized_target = self.wrap_expr( - ast.Compare(left=ast.Constant(1), ops=[opt_operators[op]], comparators=[ast.Constant(value=(1,))]) - ) - - with self.subTest( - result_code=result_code, - non_optimized_target=non_optimized_target, - optimized_target=optimized_target - ): - self.assert_ast(result_code, non_optimized_target, optimized_target) - - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags def test_folding_format(self): code = "'%s' % (a,)" @@ -3483,158 +3753,122 @@ def test_folding_format(self): self.assert_ast(code, non_optimized_target, optimized_target) + def test_folding_match_case_allowed_expressions(self): + def get_match_case_values(node): + result = [] + if isinstance(node, ast.Constant): + result.append(node.value) + elif isinstance(node, ast.MatchValue): + result.extend(get_match_case_values(node.value)) + elif isinstance(node, ast.MatchMapping): + for key in node.keys: + result.extend(get_match_case_values(key)) + elif isinstance(node, ast.MatchSequence): + for pat in node.patterns: + result.extend(get_match_case_values(pat)) + else: + self.fail(f"Unexpected node {node}") + return result - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags - def test_folding_tuple(self): - code = "(1,)" + tests = [ + ("-0", [0]), + ("-0.1", [-0.1]), + ("-0j", [complex(0, 0)]), + ("-0.1j", [complex(0, -0.1)]), + ("1 + 2j", [complex(1, 2)]), + ("1 - 2j", [complex(1, -2)]), + ("1.1 + 2.1j", [complex(1.1, 2.1)]), + ("1.1 - 2.1j", [complex(1.1, -2.1)]), + ("-0 + 1j", [complex(0, 1)]), + ("-0 - 1j", [complex(0, -1)]), + ("-0.1 + 1.1j", [complex(-0.1, 1.1)]), + ("-0.1 - 1.1j", [complex(-0.1, -1.1)]), + ("{-0: 0}", [0]), + ("{-0.1: 0}", [-0.1]), + ("{-0j: 0}", [complex(0, 0)]), + ("{-0.1j: 0}", [complex(0, -0.1)]), + ("{1 + 2j: 0}", [complex(1, 2)]), + ("{1 - 2j: 0}", [complex(1, -2)]), + ("{1.1 + 2.1j: 0}", [complex(1.1, 2.1)]), + ("{1.1 - 2.1j: 0}", [complex(1.1, -2.1)]), + ("{-0 + 1j: 0}", [complex(0, 1)]), + ("{-0 - 1j: 0}", [complex(0, -1)]), + ("{-0.1 + 1.1j: 0}", [complex(-0.1, 1.1)]), + ("{-0.1 - 1.1j: 0}", [complex(-0.1, -1.1)]), + ("{-0: 0, 0 + 1j: 0, 0.1 + 1j: 0}", [0, complex(0, 1), complex(0.1, 1)]), + ("[-0, -0.1, -0j, -0.1j]", [0, -0.1, complex(0, 0), complex(0, -0.1)]), + ("[[[[-0, -0.1, -0j, -0.1j]]]]", [0, -0.1, complex(0, 0), complex(0, -0.1)]), + ("[[-0, -0.1], -0j, -0.1j]", [0, -0.1, complex(0, 0), complex(0, -0.1)]), + ("[[-0, -0.1], [-0j, -0.1j]]", [0, -0.1, complex(0, 0), complex(0, -0.1)]), + ("(-0, -0.1, -0j, -0.1j)", [0, -0.1, complex(0, 0), complex(0, -0.1)]), + ("((((-0, -0.1, -0j, -0.1j))))", [0, -0.1, complex(0, 0), complex(0, -0.1)]), + ("((-0, -0.1), -0j, -0.1j)", [0, -0.1, complex(0, 0), complex(0, -0.1)]), + ("((-0, -0.1), (-0j, -0.1j))", [0, -0.1, complex(0, 0), complex(0, -0.1)]), + ] + for match_expr, constants in tests: + with self.subTest(match_expr): + src = f"match 0:\n\t case {match_expr}: pass" + tree = ast.parse(src, optimize=1) + match_stmt = tree.body[0] + case = match_stmt.cases[0] + values = get_match_case_values(case.pattern) + self.assertListEqual(constants, values) + + def test_match_case_not_folded_in_unoptimized_ast(self): + src = textwrap.dedent(""" + match a: + case 1+2j: + pass + """) - non_optimized_target = self.wrap_expr(ast.Tuple(elts=[ast.Constant(1)])) - optimized_target = self.wrap_expr(ast.Constant(value=(1,))) + unfolded = "MatchValue(value=BinOp(left=Constant(value=1), op=Add(), right=Constant(value=2j))" + folded = "MatchValue(value=Constant(value=(1+2j)))" + for optval in (0, 1, 2): + self.assertIn(folded if optval else unfolded, ast.dump(ast.parse(src, optimize=optval))) - self.assert_ast(code, non_optimized_target, optimized_target) + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags + def test_folding_binop(self): + return super().test_folding_binop() - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags def test_folding_comparator(self): - code = "1 %s %s1%s" - operators = [("in", ast.In()), ("not in", ast.NotIn())] - braces = [ - ("[", "]", ast.List, (1,)), - ("{", "}", ast.Set, frozenset({1})), - ] - for left, right, non_optimized_comparator, optimized_comparator in braces: - for op, node in operators: - non_optimized_target = self.wrap_expr(ast.Compare( - left=ast.Constant(1), ops=[node], - comparators=[non_optimized_comparator(elts=[ast.Constant(1)])] - )) - optimized_target = self.wrap_expr(ast.Compare( - left=ast.Constant(1), ops=[node], - comparators=[ast.Constant(value=optimized_comparator)] - )) - self.assert_ast(code % (op, left, right), non_optimized_target, optimized_target) - - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags - def test_folding_iter(self): - code = "for _ in %s1%s: pass" - braces = [ - ("[", "]", ast.List, (1,)), - ("{", "}", ast.Set, frozenset({1})), - ] - - for left, right, ast_cls, optimized_iter in braces: - non_optimized_target = self.wrap_statement(ast.For( - target=ast.Name(id="_", ctx=ast.Store()), - iter=ast_cls(elts=[ast.Constant(1)]), - body=[ast.Pass()] - )) - optimized_target = self.wrap_statement(ast.For( - target=ast.Name(id="_", ctx=ast.Store()), - iter=ast.Constant(value=optimized_iter), - body=[ast.Pass()] - )) - - self.assert_ast(code % (left, right), non_optimized_target, optimized_target) - - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags - def test_folding_subscript(self): - code = "(1,)[0]" - - non_optimized_target = self.wrap_expr( - ast.Subscript(value=ast.Tuple(elts=[ast.Constant(value=1)]), slice=ast.Constant(value=0)) - ) - optimized_target = self.wrap_expr(ast.Constant(value=1)) + return super().test_folding_comparator() - self.assert_ast(code, non_optimized_target, optimized_target) + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags + def test_folding_iter(self): + return super().test_folding_iter() - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags - def test_folding_type_param_in_function_def(self): - code = "def foo[%s = 1 + 1](): pass" + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags + def test_folding_not(self): + return super().test_folding_not() - unoptimized_binop = self.create_binop("+") - unoptimized_type_params = [ - ("T", "T", ast.TypeVar), - ("**P", "P", ast.ParamSpec), - ("*Ts", "Ts", ast.TypeVarTuple), - ] + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags + def test_folding_subscript(self): + return super().test_folding_subscript() - for type, name, type_param in unoptimized_type_params: - result_code = code % type - optimized_target = self.wrap_statement( - ast.FunctionDef( - name='foo', - args=ast.arguments(), - body=[ast.Pass()], - type_params=[type_param(name=name, default_value=ast.Constant(2))] - ) - ) - non_optimized_target = self.wrap_statement( - ast.FunctionDef( - name='foo', - args=ast.arguments(), - body=[ast.Pass()], - type_params=[type_param(name=name, default_value=unoptimized_binop)] - ) - ) - self.assert_ast(result_code, non_optimized_target, optimized_target) + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags + def test_folding_tuple(self): + return super().test_folding_tuple() - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags def test_folding_type_param_in_class_def(self): - code = "class foo[%s = 1 + 1]: pass" - - unoptimized_binop = self.create_binop("+") - unoptimized_type_params = [ - ("T", "T", ast.TypeVar), - ("**P", "P", ast.ParamSpec), - ("*Ts", "Ts", ast.TypeVarTuple), - ] + return super().test_folding_type_param_in_class_def() - for type, name, type_param in unoptimized_type_params: - result_code = code % type - optimized_target = self.wrap_statement( - ast.ClassDef( - name='foo', - body=[ast.Pass()], - type_params=[type_param(name=name, default_value=ast.Constant(2))] - ) - ) - non_optimized_target = self.wrap_statement( - ast.ClassDef( - name='foo', - body=[ast.Pass()], - type_params=[type_param(name=name, default_value=unoptimized_binop)] - ) - ) - self.assert_ast(result_code, non_optimized_target, optimized_target) + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags + def test_folding_type_param_in_function_def(self): + return super().test_folding_type_param_in_function_def() - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags def test_folding_type_param_in_type_alias(self): - code = "type foo[%s = 1 + 1] = 1" - - unoptimized_binop = self.create_binop("+") - unoptimized_type_params = [ - ("T", "T", ast.TypeVar), - ("**P", "P", ast.ParamSpec), - ("*Ts", "Ts", ast.TypeVarTuple), - ] + return super().test_folding_type_param_in_type_alias() - for type, name, type_param in unoptimized_type_params: - result_code = code % type - optimized_target = self.wrap_statement( - ast.TypeAlias( - name=ast.Name(id='foo', ctx=ast.Store()), - type_params=[type_param(name=name, default_value=ast.Constant(2))], - value=ast.Constant(value=1), - ) - ) - non_optimized_target = self.wrap_statement( - ast.TypeAlias( - name=ast.Name(id='foo', ctx=ast.Store()), - type_params=[type_param(name=name, default_value=unoptimized_binop)], - value=ast.Constant(value=1), - ) - ) - self.assert_ast(result_code, non_optimized_target, optimized_target) + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags + def test_folding_unaryop(self): + return super().test_folding_unaryop() -if __name__ == "__main__": +if __name__ == '__main__': + if len(sys.argv) > 1 and sys.argv[1] == '--snapshot-update': + ast_repr_update_snapshots() + sys.exit(0) unittest.main() diff --git a/Lib/test/test_ast/utils.py b/Lib/test/test_ast/utils.py index 145e89ee94e..e7054f3f710 100644 --- a/Lib/test/test_ast/utils.py +++ b/Lib/test/test_ast/utils.py @@ -1,5 +1,5 @@ def to_tuple(t): - if t is None or isinstance(t, (str, int, complex, float, bytes)) or t is Ellipsis: + if t is None or isinstance(t, (str, int, complex, float, bytes, tuple)) or t is Ellipsis: return t elif isinstance(t, list): return [to_tuple(e) for e in t] diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index 45d220e3e02..181476e0989 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -4,10 +4,12 @@ import contextlib from test.support.import_helper import import_module -from test.support import gc_collect +from test.support import gc_collect, requires_working_socket asyncio = import_module("asyncio") +requires_working_socket(module=True) + _no_default = object() @@ -375,6 +377,178 @@ async def async_gen_wrapper(): self.compare_generators(sync_gen_wrapper(), async_gen_wrapper()) + def test_async_gen_exception_12(self): + async def gen(): + with self.assertWarnsRegex(RuntimeWarning, + f"coroutine method 'asend' of '{gen.__qualname__}' " + f"was never awaited"): + await anext(me) + yield 123 + + me = gen() + ai = me.__aiter__() + an = ai.__anext__() + + with self.assertRaisesRegex(RuntimeError, + r'anext\(\): asynchronous generator is already running'): + an.__next__() + + with self.assertRaisesRegex(RuntimeError, + r"cannot reuse already awaited __anext__\(\)/asend\(\)"): + an.send(None) + + def test_async_gen_asend_throw_concurrent_with_send(self): + import types + + @types.coroutine + def _async_yield(v): + return (yield v) + + class MyExc(Exception): + pass + + async def agenfn(): + while True: + try: + await _async_yield(None) + except MyExc: + pass + return + yield + + + agen = agenfn() + gen = agen.asend(None) + gen.send(None) + gen2 = agen.asend(None) + + with self.assertRaisesRegex(RuntimeError, + r'anext\(\): asynchronous generator is already running'): + gen2.throw(MyExc) + + with self.assertRaisesRegex(RuntimeError, + r"cannot reuse already awaited __anext__\(\)/asend\(\)"): + gen2.send(None) + + def test_async_gen_athrow_throw_concurrent_with_send(self): + import types + + @types.coroutine + def _async_yield(v): + return (yield v) + + class MyExc(Exception): + pass + + async def agenfn(): + while True: + try: + await _async_yield(None) + except MyExc: + pass + return + yield + + + agen = agenfn() + gen = agen.asend(None) + gen.send(None) + gen2 = agen.athrow(MyExc) + + with self.assertRaisesRegex(RuntimeError, + r'athrow\(\): asynchronous generator is already running'): + gen2.throw(MyExc) + + with self.assertRaisesRegex(RuntimeError, + r"cannot reuse already awaited aclose\(\)/athrow\(\)"): + gen2.send(None) + + def test_async_gen_asend_throw_concurrent_with_throw(self): + import types + + @types.coroutine + def _async_yield(v): + return (yield v) + + class MyExc(Exception): + pass + + async def agenfn(): + try: + yield + except MyExc: + pass + while True: + try: + await _async_yield(None) + except MyExc: + pass + + + agen = agenfn() + with self.assertRaises(StopIteration): + agen.asend(None).send(None) + + gen = agen.athrow(MyExc) + gen.throw(MyExc) + gen2 = agen.asend(MyExc) + + with self.assertRaisesRegex(RuntimeError, + r'anext\(\): asynchronous generator is already running'): + gen2.throw(MyExc) + + with self.assertRaisesRegex(RuntimeError, + r"cannot reuse already awaited __anext__\(\)/asend\(\)"): + gen2.send(None) + + def test_async_gen_athrow_throw_concurrent_with_throw(self): + import types + + @types.coroutine + def _async_yield(v): + return (yield v) + + class MyExc(Exception): + pass + + async def agenfn(): + try: + yield + except MyExc: + pass + while True: + try: + await _async_yield(None) + except MyExc: + pass + + agen = agenfn() + with self.assertRaises(StopIteration): + agen.asend(None).send(None) + + gen = agen.athrow(MyExc) + gen.throw(MyExc) + gen2 = agen.athrow(None) + + with self.assertRaisesRegex(RuntimeError, + r'athrow\(\): asynchronous generator is already running'): + gen2.throw(MyExc) + + with self.assertRaisesRegex(RuntimeError, + r"cannot reuse already awaited aclose\(\)/athrow\(\)"): + gen2.send(None) + + def test_async_gen_3_arg_deprecation_warning(self): + async def gen(): + yield 123 + + with self.assertWarns(DeprecationWarning): + x = gen().athrow(GeneratorExit, GeneratorExit(), None) + with self.assertRaises(GeneratorExit): + x.send(None) + del x + gc_collect() + def test_async_gen_api_01(self): async def gen(): yield 123 @@ -393,8 +567,57 @@ async def gen(): self.assertIsInstance(g.ag_frame, types.FrameType) self.assertFalse(g.ag_running) self.assertIsInstance(g.ag_code, types.CodeType) + aclose = g.aclose() + self.assertTrue(inspect.isawaitable(aclose)) + aclose.close() + + def test_async_gen_asend_close_runtime_error(self): + import types + + @types.coroutine + def _async_yield(v): + return (yield v) - self.assertTrue(inspect.isawaitable(g.aclose())) + async def agenfn(): + try: + await _async_yield(None) + except GeneratorExit: + await _async_yield(None) + return + yield + + agen = agenfn() + gen = agen.asend(None) + gen.send(None) + with self.assertRaisesRegex(RuntimeError, "coroutine ignored GeneratorExit"): + gen.close() + + def test_async_gen_athrow_close_runtime_error(self): + import types + + @types.coroutine + def _async_yield(v): + return (yield v) + + class MyExc(Exception): + pass + + async def agenfn(): + try: + yield + except MyExc: + try: + await _async_yield(None) + except GeneratorExit: + await _async_yield(None) + + agen = agenfn() + with self.assertRaises(StopIteration): + agen.asend(None).send(None) + gen = agen.athrow(MyExc) + gen.send(None) + with self.assertRaisesRegex(RuntimeError, "coroutine ignored GeneratorExit"): + gen.close() class AsyncGenAsyncioTest(unittest.TestCase): @@ -406,7 +629,7 @@ def setUp(self): def tearDown(self): self.loop.close() self.loop = None - asyncio.set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) def check_async_iterator_anext(self, ait_class): with self.subTest(anext="pure-Python"): @@ -648,7 +871,7 @@ def test1(anext): agen = agenfn() with contextlib.closing(anext(agen, "default").__await__()) as g: self.assertEqual(g.send(None), 1) - self.assertEqual(g.throw(MyError, MyError(), None), 2) + self.assertEqual(g.throw(MyError()), 2) try: g.send(None) except StopIteration as e: @@ -661,9 +884,9 @@ def test2(anext): agen = agenfn() with contextlib.closing(anext(agen, "default").__await__()) as g: self.assertEqual(g.send(None), 1) - self.assertEqual(g.throw(MyError, MyError(), None), 2) + self.assertEqual(g.throw(MyError()), 2) with self.assertRaises(MyError): - g.throw(MyError, MyError(), None) + g.throw(MyError()) def test3(anext): agen = agenfn() @@ -690,9 +913,9 @@ async def agenfn(): agen = agenfn() with contextlib.closing(anext(agen, "default").__await__()) as g: self.assertEqual(g.send(None), 10) - self.assertEqual(g.throw(MyError, MyError(), None), 20) + self.assertEqual(g.throw(MyError()), 20) with self.assertRaisesRegex(MyError, 'val'): - g.throw(MyError, MyError('val'), None) + g.throw(MyError('val')) def test5(anext): @types.coroutine @@ -711,7 +934,7 @@ async def agenfn(): with contextlib.closing(anext(agen, "default").__await__()) as g: self.assertEqual(g.send(None), 10) with self.assertRaisesRegex(StopIteration, 'default'): - g.throw(MyError, MyError(), None) + g.throw(MyError()) def test6(anext): @types.coroutine @@ -726,7 +949,7 @@ async def agenfn(): agen = agenfn() with contextlib.closing(anext(agen, "default").__await__()) as g: with self.assertRaises(MyError): - g.throw(MyError, MyError(), None) + g.throw(MyError()) def run_test(test): with self.subTest('pure-Python anext()'): @@ -929,6 +1152,43 @@ async def run(): self.loop.run_until_complete(run()) + def test_async_gen_asyncio_anext_tuple_no_exceptions(self): + # StopAsyncIteration exceptions should be cleared. + # See: https://github.com/python/cpython/issues/128078. + + async def foo(): + if False: + yield (1, 2) + + async def run(): + it = foo().__aiter__() + with self.assertRaises(StopAsyncIteration): + await it.__anext__() + res = await anext(it, ('a', 'b')) + self.assertTupleEqual(res, ('a', 'b')) + + self.loop.run_until_complete(run()) + + def test_sync_anext_raises_exception(self): + # See: https://github.com/python/cpython/issues/131670 + msg = 'custom' + for exc_type in [ + StopAsyncIteration, + StopIteration, + ValueError, + Exception, + ]: + exc = exc_type(msg) + with self.subTest(exc=exc): + class A: + def __anext__(self): + raise exc + + with self.assertRaisesRegex(exc_type, msg): + anext(A()) + with self.assertRaisesRegex(exc_type, msg): + anext(A(), 1) + def test_async_gen_asyncio_anext_stopiteration(self): async def foo(): try: @@ -1035,8 +1295,7 @@ async def gen(): while True: yield 1 finally: - await asyncio.sleep(0.01) - await asyncio.sleep(0.01) + await asyncio.sleep(0) DONE = 1 async def run(): @@ -1046,7 +1305,10 @@ async def run(): del g gc_collect() # For PyPy or other GCs. - await asyncio.sleep(0.1) + # Starts running the aclose task + await asyncio.sleep(0) + # For asyncio.sleep(0) in finally block + await asyncio.sleep(0) self.loop.run_until_complete(run()) self.assertEqual(DONE, 1) @@ -1539,6 +1801,8 @@ async def main(): self.assertIsInstance(message['exception'], ZeroDivisionError) self.assertIn('unhandled exception during asyncio.run() shutdown', message['message']) + del message, messages + gc_collect() def test_async_gen_expression_01(self): async def arange(n): @@ -1556,21 +1820,35 @@ async def run(): res = self.loop.run_until_complete(run()) self.assertEqual(res, [i * 2 for i in range(10)]) - # TODO: RUSTPYTHON: async for gen expression compilation - # def test_async_gen_expression_02(self): - # async def wrap(n): - # await asyncio.sleep(0.01) - # return n + def test_async_gen_expression_02(self): + async def wrap(n): + await asyncio.sleep(0.01) + return n - # def make_arange(n): - # # This syntax is legal starting with Python 3.7 - # return (i * 2 for i in range(n) if await wrap(i)) + def make_arange(n): + # This syntax is legal starting with Python 3.7 + return (i * 2 for i in range(n) if await wrap(i)) - # async def run(): - # return [i async for i in make_arange(10)] + async def run(): + return [i async for i in make_arange(10)] + + res = self.loop.run_until_complete(run()) + self.assertEqual(res, [i * 2 for i in range(1, 10)]) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: __aiter__ + def test_async_gen_expression_incorrect(self): + async def ag(): + yield 42 - # res = self.loop.run_until_complete(run()) - # self.assertEqual(res, [i * 2 for i in range(1, 10)]) + async def run(arg): + (x async for x in arg) + + err_msg_async = "'async for' requires an object with " \ + "__aiter__ method, got .*" + + self.loop.run_until_complete(run(ag())) + with self.assertRaisesRegex(TypeError, err_msg_async): + self.loop.run_until_complete(run(None)) def test_asyncgen_nonstarted_hooks_are_cancellable(self): # See https://bugs.python.org/issue38013 @@ -1593,6 +1871,7 @@ async def main(): asyncio.run(main()) self.assertEqual([], messages) + gc_collect() def test_async_gen_await_same_anext_coro_twice(self): async def async_iterate(): @@ -1630,6 +1909,62 @@ async def run(): self.loop.run_until_complete(run()) + def test_async_gen_throw_same_aclose_coro_twice(self): + async def async_iterate(): + yield 1 + yield 2 + + it = async_iterate() + nxt = it.aclose() + with self.assertRaises(StopIteration): + nxt.throw(GeneratorExit) + + with self.assertRaisesRegex( + RuntimeError, + r"cannot reuse already awaited aclose\(\)/athrow\(\)" + ): + nxt.throw(GeneratorExit) + + def test_async_gen_throw_custom_same_aclose_coro_twice(self): + async def async_iterate(): + yield 1 + yield 2 + + it = async_iterate() + + class MyException(Exception): + pass + + nxt = it.aclose() + with self.assertRaises(MyException): + nxt.throw(MyException) + + with self.assertRaisesRegex( + RuntimeError, + r"cannot reuse already awaited aclose\(\)/athrow\(\)" + ): + nxt.throw(MyException) + + def test_async_gen_throw_custom_same_athrow_coro_twice(self): + async def async_iterate(): + yield 1 + yield 2 + + it = async_iterate() + + class MyException(Exception): + pass + + nxt = it.athrow(MyException) + with self.assertRaises(MyException): + nxt.throw(MyException) + + with self.assertRaisesRegex( + RuntimeError, + r"cannot reuse already awaited aclose\(\)/athrow\(\)" + ): + nxt.throw(MyException) + def test_async_gen_aclose_twice_with_different_coros(self): # Regression test for https://bugs.python.org/issue39606 async def async_iterate(): @@ -1672,5 +2007,109 @@ async def run(): self.loop.run_until_complete(run()) +class TestUnawaitedWarnings(unittest.TestCase): + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: RuntimeWarning not triggered + def test_asend(self): + async def gen(): + yield 1 + + # gh-113753: asend objects allocated from a free-list should warn. + # Ensure there is a finalized 'asend' object ready to be reused. + try: + g = gen() + g.asend(None).send(None) + except StopIteration: + pass + + msg = f"coroutine method 'asend' of '{gen.__qualname__}' was never awaited" + with self.assertWarnsRegex(RuntimeWarning, msg): + g = gen() + g.asend(None) + gc_collect() + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: RuntimeWarning not triggered + def test_athrow(self): + async def gen(): + yield 1 + + msg = f"coroutine method 'athrow' of '{gen.__qualname__}' was never awaited" + with self.assertWarnsRegex(RuntimeWarning, msg): + g = gen() + g.athrow(RuntimeError) + gc_collect() + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: RuntimeWarning not triggered + def test_aclose(self): + async def gen(): + yield 1 + + msg = f"coroutine method 'aclose' of '{gen.__qualname__}' was never awaited" + with self.assertWarnsRegex(RuntimeWarning, msg): + g = gen() + g.aclose() + gc_collect() + + def test_aclose_throw(self): + async def gen(): + return + yield + + class MyException(Exception): + pass + + g = gen() + with self.assertRaises(MyException): + g.aclose().throw(MyException) + + del g + gc_collect() # does not warn unawaited + + def test_asend_send_already_running(self): + @types.coroutine + def _async_yield(v): + return (yield v) + + async def agenfn(): + while True: + await _async_yield(1) + return + yield + + agen = agenfn() + gen = agen.asend(None) + gen.send(None) + gen2 = agen.asend(None) + + with self.assertRaisesRegex(RuntimeError, + r'anext\(\): asynchronous generator is already running'): + gen2.send(None) + + del gen2 + gc_collect() # does not warn unawaited + + + def test_athrow_send_already_running(self): + @types.coroutine + def _async_yield(v): + return (yield v) + + async def agenfn(): + while True: + await _async_yield(1) + return + yield + + agen = agenfn() + gen = agen.asend(None) + gen.send(None) + gen2 = agen.athrow(Exception) + + with self.assertRaisesRegex(RuntimeError, + r'athrow\(\): asynchronous generator is already running'): + gen2.send(None) + + del gen2 + gc_collect() # does not warn unawaited + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_asyncio/functional.py b/Lib/test/test_asyncio/functional.py index d19c7a612cc..96dc9ab4401 100644 --- a/Lib/test/test_asyncio/functional.py +++ b/Lib/test/test_asyncio/functional.py @@ -217,16 +217,15 @@ def stop(self): pass finally: super().stop() - - def run(self): - try: - with self._sock: - self._sock.setblocking(False) - self._run() - finally: + self._sock.close() self._s1.close() self._s2.close() + + def run(self): + self._sock.setblocking(False) + self._run() + def _run(self): while self._active: if self._clients >= self._max_clients: diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py index 870be745098..92895bbb420 100644 --- a/Lib/test/test_asyncio/test_base_events.py +++ b/Lib/test/test_asyncio/test_base_events.py @@ -29,7 +29,7 @@ class CustomError(Exception): def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) def mock_socket_module(): @@ -1019,8 +1019,7 @@ async def iter_one(): asyncio.create_task(iter_one()) return status - # TODO: RUSTPYTHON - GC doesn't finalize async generators - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; - GC doesn't finalize async generators def test_asyncgen_finalization_by_gc(self): # Async generators should be finalized when garbage collected. self.loop._process_events = mock.Mock() @@ -1036,8 +1035,7 @@ def test_asyncgen_finalization_by_gc(self): test_utils.run_briefly(self.loop) self.assertTrue(status['finalized']) - # TODO: RUSTPYTHON - GC doesn't finalize async generators - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; - GC doesn't finalize async generators def test_asyncgen_finalization_by_gc_in_other_thread(self): # Python issue 34769: If garbage collector runs in another # thread, async generators will not finalize in debug @@ -1413,7 +1411,7 @@ def getaddrinfo_task(*args, **kwds): with self.assertRaises(OSError) as cm: self.loop.run_until_complete(coro) - self.assertTrue(str(cm.exception).startswith('Multiple exceptions: ')) + self.assertStartsWith(str(cm.exception), 'Multiple exceptions: ') self.assertTrue(m_socket.socket.return_value.close.called) coro = self.loop.create_connection( diff --git a/Lib/test/test_asyncio/test_buffered_proto.py b/Lib/test/test_asyncio/test_buffered_proto.py index f24e363ebfc..6d3edcc36f5 100644 --- a/Lib/test/test_asyncio/test_buffered_proto.py +++ b/Lib/test/test_asyncio/test_buffered_proto.py @@ -5,7 +5,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class ReceiveStuffProto(asyncio.BufferedProtocol): diff --git a/Lib/test/test_asyncio/test_context.py b/Lib/test/test_asyncio/test_context.py index 6b80721873d..f85f39839cb 100644 --- a/Lib/test/test_asyncio/test_context.py +++ b/Lib/test/test_asyncio/test_context.py @@ -4,7 +4,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) @unittest.skipUnless(decimal.HAVE_CONTEXTVAR, "decimal is built with a thread-local context") diff --git a/Lib/test/test_asyncio/test_eager_task_factory.py b/Lib/test/test_asyncio/test_eager_task_factory.py index 179a6e44b59..0561b54a3f1 100644 --- a/Lib/test/test_asyncio/test_eager_task_factory.py +++ b/Lib/test/test_asyncio/test_eager_task_factory.py @@ -13,7 +13,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class EagerTaskFactoryLoopTests: @@ -263,19 +263,62 @@ async def run(): self.run_coro(run()) + def test_eager_start_false(self): + name = None + + async def asyncfn(): + nonlocal name + name = asyncio.current_task().get_name() + + async def main(): + t = asyncio.get_running_loop().create_task( + asyncfn(), eager_start=False, name="example" + ) + self.assertFalse(t.done()) + self.assertIsNone(name) + await t + self.assertEqual(name, "example") + + self.run_coro(main()) + class PyEagerTaskFactoryLoopTests(EagerTaskFactoryLoopTests, test_utils.TestCase): Task = tasks._PyTask + def setUp(self): + self._all_tasks = asyncio.all_tasks + self._current_task = asyncio.current_task + asyncio.current_task = asyncio.tasks.current_task = asyncio.tasks._py_current_task + asyncio.all_tasks = asyncio.tasks.all_tasks = asyncio.tasks._py_all_tasks + return super().setUp() + + def tearDown(self): + asyncio.current_task = asyncio.tasks.current_task = self._current_task + asyncio.all_tasks = asyncio.tasks.all_tasks = self._all_tasks + return super().tearDown() + + @unittest.skipUnless(hasattr(tasks, '_CTask'), 'requires the C _asyncio module') class CEagerTaskFactoryLoopTests(EagerTaskFactoryLoopTests, test_utils.TestCase): Task = getattr(tasks, '_CTask', None) + def setUp(self): + self._current_task = asyncio.current_task + self._all_tasks = asyncio.all_tasks + asyncio.current_task = asyncio.tasks.current_task = asyncio.tasks._c_current_task + asyncio.all_tasks = asyncio.tasks.all_tasks = asyncio.tasks._c_all_tasks + return super().setUp() + + def tearDown(self): + asyncio.current_task = asyncio.tasks.current_task = self._current_task + asyncio.all_tasks = asyncio.tasks.all_tasks = self._all_tasks + return super().tearDown() + def test_issue105987(self): code = """if 1: - from _asyncio import _swap_current_task + from _asyncio import _swap_current_task, _set_running_loop class DummyTask: pass @@ -284,6 +327,7 @@ class DummyLoop: pass l = DummyLoop() + _set_running_loop(l) _swap_current_task(l, DummyTask()) t = _swap_current_task(l, None) """ @@ -400,31 +444,102 @@ class BaseEagerTaskFactoryTests(BaseTaskCountingTests): class NonEagerTests(BaseNonEagerTaskFactoryTests, test_utils.TestCase): - Task = asyncio.Task + Task = asyncio.tasks._CTask + def setUp(self): + self._current_task = asyncio.current_task + asyncio.current_task = asyncio.tasks.current_task = asyncio.tasks._c_current_task + return super().setUp() + + def tearDown(self): + asyncio.current_task = asyncio.tasks.current_task = self._current_task + return super().tearDown() class EagerTests(BaseEagerTaskFactoryTests, test_utils.TestCase): - Task = asyncio.Task + Task = asyncio.tasks._CTask + + def setUp(self): + self._current_task = asyncio.current_task + asyncio.current_task = asyncio.tasks.current_task = asyncio.tasks._c_current_task + return super().setUp() + + def tearDown(self): + asyncio.current_task = asyncio.tasks.current_task = self._current_task + return super().tearDown() class NonEagerPyTaskTests(BaseNonEagerTaskFactoryTests, test_utils.TestCase): Task = tasks._PyTask + def setUp(self): + self._current_task = asyncio.current_task + asyncio.current_task = asyncio.tasks.current_task = asyncio.tasks._py_current_task + return super().setUp() + + def tearDown(self): + asyncio.current_task = asyncio.tasks.current_task = self._current_task + return super().tearDown() + class EagerPyTaskTests(BaseEagerTaskFactoryTests, test_utils.TestCase): Task = tasks._PyTask + def setUp(self): + self._current_task = asyncio.current_task + asyncio.current_task = asyncio.tasks.current_task = asyncio.tasks._py_current_task + return super().setUp() + + def tearDown(self): + asyncio.current_task = asyncio.tasks.current_task = self._current_task + return super().tearDown() @unittest.skipUnless(hasattr(tasks, '_CTask'), 'requires the C _asyncio module') class NonEagerCTaskTests(BaseNonEagerTaskFactoryTests, test_utils.TestCase): Task = getattr(tasks, '_CTask', None) + def setUp(self): + self._current_task = asyncio.current_task + asyncio.current_task = asyncio.tasks.current_task = asyncio.tasks._c_current_task + return super().setUp() + + def tearDown(self): + asyncio.current_task = asyncio.tasks.current_task = self._current_task + return super().tearDown() + @unittest.skipUnless(hasattr(tasks, '_CTask'), 'requires the C _asyncio module') class EagerCTaskTests(BaseEagerTaskFactoryTests, test_utils.TestCase): Task = getattr(tasks, '_CTask', None) + def setUp(self): + self._current_task = asyncio.current_task + asyncio.current_task = asyncio.tasks.current_task = asyncio.tasks._c_current_task + return super().setUp() + + def tearDown(self): + asyncio.current_task = asyncio.tasks.current_task = self._current_task + return super().tearDown() + + +class DefaultTaskFactoryEagerStart(test_utils.TestCase): + def test_eager_start_true_with_default_factory(self): + name = None + + async def asyncfn(): + nonlocal name + name = asyncio.current_task().get_name() + + async def main(): + t = asyncio.get_running_loop().create_task( + asyncfn(), eager_start=True, name="example" + ) + self.assertTrue(t.done()) + self.assertEqual(name, "example") + await t + + asyncio.run(main(), loop_factory=asyncio.EventLoop) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index 2cbb96da696..b60c7452f3f 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -23,7 +23,6 @@ import unittest from unittest import mock import weakref -import warnings if sys.platform not in ('win32', 'vxworks'): import tty @@ -38,9 +37,8 @@ from test.support import threading_helper from test.support import ALWAYS_EQ, LARGEST, SMALLEST - def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) def broken_unix_getsockname(): @@ -275,8 +273,7 @@ def tearDown(self): support.gc_collect() super().tearDown() - # TODO: RUSTPYTHON - RuntimeWarning for unawaited coroutine not triggered - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; - RuntimeWarning for unawaited coroutine not triggered def test_run_until_complete_nesting(self): async def coro1(): await asyncio.sleep(0) @@ -356,6 +353,124 @@ def run_in_thread(): t.join() self.assertEqual(results, ['hello', 'world']) + def test_call_soon_threadsafe_handle_block_check_cancelled(self): + results = [] + + callback_started = threading.Event() + callback_finished = threading.Event() + def callback(arg): + callback_started.set() + results.append(arg) + time.sleep(1) + callback_finished.set() + + def run_in_thread(): + handle = self.loop.call_soon_threadsafe(callback, 'hello') + self.assertIsInstance(handle, events._ThreadSafeHandle) + callback_started.wait() + # callback started so it should block checking for cancellation + # until it finishes + self.assertFalse(handle.cancelled()) + self.assertTrue(callback_finished.is_set()) + self.loop.call_soon_threadsafe(self.loop.stop) + + t = threading.Thread(target=run_in_thread) + t.start() + + self.loop.run_forever() + t.join() + self.assertEqual(results, ['hello']) + + def test_call_soon_threadsafe_handle_block_cancellation(self): + results = [] + + callback_started = threading.Event() + callback_finished = threading.Event() + def callback(arg): + callback_started.set() + results.append(arg) + time.sleep(1) + callback_finished.set() + + def run_in_thread(): + handle = self.loop.call_soon_threadsafe(callback, 'hello') + self.assertIsInstance(handle, events._ThreadSafeHandle) + callback_started.wait() + # callback started so it cannot be cancelled from other thread until + # it finishes + handle.cancel() + self.assertTrue(callback_finished.is_set()) + self.loop.call_soon_threadsafe(self.loop.stop) + + t = threading.Thread(target=run_in_thread) + t.start() + + self.loop.run_forever() + t.join() + self.assertEqual(results, ['hello']) + + def test_call_soon_threadsafe_handle_cancel_same_thread(self): + results = [] + callback_started = threading.Event() + callback_finished = threading.Event() + + fut = concurrent.futures.Future() + def callback(arg): + callback_started.set() + handle = fut.result() + handle.cancel() + results.append(arg) + callback_finished.set() + self.loop.stop() + + def run_in_thread(): + handle = self.loop.call_soon_threadsafe(callback, 'hello') + fut.set_result(handle) + self.assertIsInstance(handle, events._ThreadSafeHandle) + callback_started.wait() + # callback cancels itself from same thread so it has no effect + # it runs to completion + self.assertTrue(handle.cancelled()) + self.assertTrue(callback_finished.is_set()) + self.loop.call_soon_threadsafe(self.loop.stop) + + t = threading.Thread(target=run_in_thread) + t.start() + + self.loop.run_forever() + t.join() + self.assertEqual(results, ['hello']) + + def test_call_soon_threadsafe_handle_cancel_other_thread(self): + results = [] + ev = threading.Event() + + callback_finished = threading.Event() + def callback(arg): + results.append(arg) + callback_finished.set() + self.loop.stop() + + def run_in_thread(): + handle = self.loop.call_soon_threadsafe(callback, 'hello') + # handle can be cancelled from other thread if not started yet + self.assertIsInstance(handle, events._ThreadSafeHandle) + handle.cancel() + self.assertTrue(handle.cancelled()) + self.assertFalse(callback_finished.is_set()) + ev.set() + self.loop.call_soon_threadsafe(self.loop.stop) + + # block the main loop until the callback is added and cancelled in the + # other thread + self.loop.call_soon(ev.wait) + t = threading.Thread(target=run_in_thread) + t.start() + self.loop.run_forever() + t.join() + self.assertEqual(results, []) + self.assertFalse(callback_finished.is_set()) + def test_call_soon_threadsafe_same_thread(self): results = [] @@ -445,8 +560,7 @@ def writer(data): r.close() self.assertEqual(read, data) - # TODO: RUSTPYTHON - signal handler implementation differs - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; - signal handler implementation differs @unittest.skipUnless(hasattr(signal, 'SIGKILL'), 'No SIGKILL') def test_add_signal_handler(self): caught = 0 @@ -1174,8 +1288,7 @@ def test_create_unix_server_ssl_verified(self): server.close() self.loop.run_until_complete(proto.done) - # TODO: RUSTPYTHON - SSL peer certificate format differs - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; - SSL peer certificate format differs @unittest.skipIf(ssl is None, 'No ssl module') def test_create_server_ssl_verified(self): proto = MyProto(loop=self.loop) @@ -2073,7 +2186,7 @@ def test_subprocess_stderr(self): transp.close() self.assertEqual(b'OUT:test', proto.data[1]) - self.assertTrue(proto.data[2].startswith(b'ERR:test'), proto.data[2]) + self.assertStartsWith(proto.data[2], b'ERR:test') self.assertEqual(0, proto.returncode) @support.requires_subprocess() @@ -2095,8 +2208,7 @@ def test_subprocess_stderr_redirect_to_stdout(self): stdin.write(b'test') self.loop.run_until_complete(proto.completed) - self.assertTrue(proto.data[1].startswith(b'OUT:testERR:test'), - proto.data[1]) + self.assertStartsWith(proto.data[1], b'OUT:testERR:test') self.assertEqual(b'', proto.data[2]) transp.close() @@ -2216,24 +2328,8 @@ def test_remove_fds_after_closing(self): else: import selectors - class UnixEventLoopTestsMixin(EventLoopTestsMixin): - def setUp(self): - super().setUp() - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - watcher = asyncio.SafeChildWatcher() - watcher.attach_loop(self.loop) - asyncio.set_child_watcher(watcher) - - def tearDown(self): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - asyncio.set_child_watcher(None) - super().tearDown() - - if hasattr(selectors, 'KqueueSelector'): - class KqueueEventLoopTests(UnixEventLoopTestsMixin, + class KqueueEventLoopTests(EventLoopTestsMixin, SubprocessTestsMixin, test_utils.TestCase): @@ -2258,7 +2354,7 @@ def test_write_pty(self): super().test_write_pty() if hasattr(selectors, 'EpollSelector'): - class EPollEventLoopTests(UnixEventLoopTestsMixin, + class EPollEventLoopTests(EventLoopTestsMixin, SubprocessTestsMixin, test_utils.TestCase): @@ -2266,7 +2362,7 @@ def create_event_loop(self): return asyncio.SelectorEventLoop(selectors.EpollSelector()) if hasattr(selectors, 'PollSelector'): - class PollEventLoopTests(UnixEventLoopTestsMixin, + class PollEventLoopTests(EventLoopTestsMixin, SubprocessTestsMixin, test_utils.TestCase): @@ -2274,7 +2370,7 @@ def create_event_loop(self): return asyncio.SelectorEventLoop(selectors.PollSelector()) # Should always exist. - class SelectEventLoopTests(UnixEventLoopTestsMixin, + class SelectEventLoopTests(EventLoopTestsMixin, SubprocessTestsMixin, test_utils.TestCase): @@ -2420,7 +2516,7 @@ def test_handle_repr_debug(self): self.assertRegex(repr(h), regex) def test_handle_source_traceback(self): - loop = asyncio.get_event_loop_policy().new_event_loop() + loop = asyncio.new_event_loop() loop.set_debug(True) self.set_event_loop(loop) @@ -2559,33 +2655,46 @@ def callback(*args): h1 = asyncio.TimerHandle(when, callback, (), self.loop) h2 = asyncio.TimerHandle(when, callback, (), self.loop) - # TODO: Use assertLess etc. - self.assertFalse(h1 < h2) - self.assertFalse(h2 < h1) - self.assertTrue(h1 <= h2) - self.assertTrue(h2 <= h1) - self.assertFalse(h1 > h2) - self.assertFalse(h2 > h1) - self.assertTrue(h1 >= h2) - self.assertTrue(h2 >= h1) - self.assertTrue(h1 == h2) - self.assertFalse(h1 != h2) + with self.assertRaises(AssertionError): + self.assertLess(h1, h2) + with self.assertRaises(AssertionError): + self.assertLess(h2, h1) + with self.assertRaises(AssertionError): + self.assertGreater(h1, h2) + with self.assertRaises(AssertionError): + self.assertGreater(h2, h1) + with self.assertRaises(AssertionError): + self.assertNotEqual(h1, h2) + + self.assertLessEqual(h1, h2) + self.assertLessEqual(h2, h1) + self.assertGreaterEqual(h1, h2) + self.assertGreaterEqual(h2, h1) + self.assertEqual(h1, h2) h2.cancel() - self.assertFalse(h1 == h2) + with self.assertRaises(AssertionError): + self.assertEqual(h1, h2) + self.assertNotEqual(h1, h2) h1 = asyncio.TimerHandle(when, callback, (), self.loop) h2 = asyncio.TimerHandle(when + 10.0, callback, (), self.loop) - self.assertTrue(h1 < h2) - self.assertFalse(h2 < h1) - self.assertTrue(h1 <= h2) - self.assertFalse(h2 <= h1) - self.assertFalse(h1 > h2) - self.assertTrue(h2 > h1) - self.assertFalse(h1 >= h2) - self.assertTrue(h2 >= h1) - self.assertFalse(h1 == h2) - self.assertTrue(h1 != h2) + with self.assertRaises(AssertionError): + self.assertLess(h2, h1) + with self.assertRaises(AssertionError): + self.assertLessEqual(h2, h1) + with self.assertRaises(AssertionError): + self.assertGreater(h1, h2) + with self.assertRaises(AssertionError): + self.assertGreaterEqual(h1, h2) + with self.assertRaises(AssertionError): + self.assertEqual(h1, h2) + + self.assertLess(h1, h2) + self.assertGreater(h2, h1) + self.assertLessEqual(h1, h2) + self.assertGreaterEqual(h2, h1) + self.assertNotEqual(h1, h2) h3 = asyncio.Handle(callback, (), self.loop) self.assertIs(NotImplemented, h1.__eq__(h3)) @@ -2599,19 +2708,25 @@ def callback(*args): h1 <= () with self.assertRaises(TypeError): h1 >= () - self.assertFalse(h1 == ()) - self.assertTrue(h1 != ()) - - self.assertTrue(h1 == ALWAYS_EQ) - self.assertFalse(h1 != ALWAYS_EQ) - self.assertTrue(h1 < LARGEST) - self.assertFalse(h1 > LARGEST) - self.assertTrue(h1 <= LARGEST) - self.assertFalse(h1 >= LARGEST) - self.assertFalse(h1 < SMALLEST) - self.assertTrue(h1 > SMALLEST) - self.assertFalse(h1 <= SMALLEST) - self.assertTrue(h1 >= SMALLEST) + with self.assertRaises(AssertionError): + self.assertEqual(h1, ()) + with self.assertRaises(AssertionError): + self.assertNotEqual(h1, ALWAYS_EQ) + with self.assertRaises(AssertionError): + self.assertGreater(h1, LARGEST) + with self.assertRaises(AssertionError): + self.assertGreaterEqual(h1, LARGEST) + with self.assertRaises(AssertionError): + self.assertLess(h1, SMALLEST) + with self.assertRaises(AssertionError): + self.assertLessEqual(h1, SMALLEST) + + self.assertNotEqual(h1, ()) + self.assertEqual(h1, ALWAYS_EQ) + self.assertLess(h1, LARGEST) + self.assertLessEqual(h1, LARGEST) + self.assertGreaterEqual(h1, SMALLEST) + self.assertGreater(h1, SMALLEST) class AbstractEventLoopTests(unittest.TestCase): @@ -2718,58 +2833,54 @@ async def inner(): class PolicyTests(unittest.TestCase): + def test_abstract_event_loop_policy_deprecation(self): + with self.assertWarnsRegex( + DeprecationWarning, "'asyncio.AbstractEventLoopPolicy' is deprecated"): + policy = asyncio.AbstractEventLoopPolicy() + self.assertIsInstance(policy, asyncio.AbstractEventLoopPolicy) + + def test_default_event_loop_policy_deprecation(self): + with self.assertWarnsRegex( + DeprecationWarning, "'asyncio.DefaultEventLoopPolicy' is deprecated"): + policy = asyncio.DefaultEventLoopPolicy() + self.assertIsInstance(policy, asyncio.DefaultEventLoopPolicy) + def test_event_loop_policy(self): - policy = asyncio.AbstractEventLoopPolicy() + policy = asyncio.events._AbstractEventLoopPolicy() self.assertRaises(NotImplementedError, policy.get_event_loop) self.assertRaises(NotImplementedError, policy.set_event_loop, object()) self.assertRaises(NotImplementedError, policy.new_event_loop) - self.assertRaises(NotImplementedError, policy.get_child_watcher) - self.assertRaises(NotImplementedError, policy.set_child_watcher, - object()) def test_get_event_loop(self): - policy = asyncio.DefaultEventLoopPolicy() + policy = test_utils.DefaultEventLoopPolicy() self.assertIsNone(policy._local._loop) - with self.assertWarns(DeprecationWarning) as cm: - loop = policy.get_event_loop() - self.assertEqual(cm.filename, __file__) - self.assertIsInstance(loop, asyncio.AbstractEventLoop) - self.assertIs(policy._local._loop, loop) - self.assertIs(loop, policy.get_event_loop()) - loop.close() + with self.assertRaises(RuntimeError): + loop = policy.get_event_loop() + self.assertIsNone(policy._local._loop) - def test_get_event_loop_calls_set_event_loop(self): - policy = asyncio.DefaultEventLoopPolicy() + def test_get_event_loop_does_not_call_set_event_loop(self): + policy = test_utils.DefaultEventLoopPolicy() with mock.patch.object( policy, "set_event_loop", wraps=policy.set_event_loop) as m_set_event_loop: - with self.assertWarns(DeprecationWarning) as cm: + with self.assertRaises(RuntimeError): loop = policy.get_event_loop() - self.addCleanup(loop.close) - self.assertEqual(cm.filename, __file__) - # policy._local._loop must be set through .set_event_loop() - # (the unix DefaultEventLoopPolicy needs this call to attach - # the child watcher correctly) - m_set_event_loop.assert_called_with(loop) - - loop.close() + m_set_event_loop.assert_not_called() def test_get_event_loop_after_set_none(self): - policy = asyncio.DefaultEventLoopPolicy() + policy = test_utils.DefaultEventLoopPolicy() policy.set_event_loop(None) self.assertRaises(RuntimeError, policy.get_event_loop) - # TODO: RUSTPYTHON - mock.patch doesn't work correctly with threading.current_thread - @unittest.expectedFailure @mock.patch('asyncio.events.threading.current_thread') def test_get_event_loop_thread(self, m_current_thread): def f(): - policy = asyncio.DefaultEventLoopPolicy() + policy = test_utils.DefaultEventLoopPolicy() self.assertRaises(RuntimeError, policy.get_event_loop) th = threading.Thread(target=f) @@ -2777,14 +2888,14 @@ def f(): th.join() def test_new_event_loop(self): - policy = asyncio.DefaultEventLoopPolicy() + policy = test_utils.DefaultEventLoopPolicy() loop = policy.new_event_loop() self.assertIsInstance(loop, asyncio.AbstractEventLoop) loop.close() def test_set_event_loop(self): - policy = asyncio.DefaultEventLoopPolicy() + policy = test_utils.DefaultEventLoopPolicy() old_loop = policy.new_event_loop() policy.set_event_loop(old_loop) @@ -2798,20 +2909,31 @@ def test_set_event_loop(self): old_loop.close() def test_get_event_loop_policy(self): - policy = asyncio.get_event_loop_policy() - self.assertIsInstance(policy, asyncio.AbstractEventLoopPolicy) - self.assertIs(policy, asyncio.get_event_loop_policy()) + with self.assertWarnsRegex( + DeprecationWarning, "'asyncio.get_event_loop_policy' is deprecated"): + policy = asyncio.get_event_loop_policy() + self.assertIsInstance(policy, asyncio.events._AbstractEventLoopPolicy) + self.assertIs(policy, asyncio.get_event_loop_policy()) def test_set_event_loop_policy(self): - self.assertRaises( - TypeError, asyncio.set_event_loop_policy, object()) + with self.assertWarnsRegex( + DeprecationWarning, "'asyncio.set_event_loop_policy' is deprecated"): + self.assertRaises( + TypeError, asyncio.set_event_loop_policy, object()) + + with self.assertWarnsRegex( + DeprecationWarning, "'asyncio.get_event_loop_policy' is deprecated"): + old_policy = asyncio.get_event_loop_policy() - old_policy = asyncio.get_event_loop_policy() + policy = test_utils.DefaultEventLoopPolicy() + with self.assertWarnsRegex( + DeprecationWarning, "'asyncio.set_event_loop_policy' is deprecated"): + asyncio.set_event_loop_policy(policy) - policy = asyncio.DefaultEventLoopPolicy() - asyncio.set_event_loop_policy(policy) - self.assertIs(policy, asyncio.get_event_loop_policy()) - self.assertIsNot(policy, old_policy) + with self.assertWarnsRegex( + DeprecationWarning, "'asyncio.get_event_loop_policy' is deprecated"): + self.assertIs(policy, asyncio.get_event_loop_policy()) + self.assertIsNot(policy, old_policy) class GetEventLoopTestsMixin: @@ -2821,11 +2943,16 @@ class GetEventLoopTestsMixin: get_running_loop_impl = None get_event_loop_impl = None + Task = None + Future = None + def setUp(self): self._get_running_loop_saved = events._get_running_loop self._set_running_loop_saved = events._set_running_loop self.get_running_loop_saved = events.get_running_loop self.get_event_loop_saved = events.get_event_loop + self._Task_saved = asyncio.Task + self._Future_saved = asyncio.Future events._get_running_loop = type(self)._get_running_loop_impl events._set_running_loop = type(self)._set_running_loop_impl @@ -2837,25 +2964,15 @@ def setUp(self): asyncio.get_running_loop = type(self).get_running_loop_impl asyncio.get_event_loop = type(self).get_event_loop_impl + asyncio.Task = asyncio.tasks.Task = type(self).Task + asyncio.Future = asyncio.futures.Future = type(self).Future super().setUp() self.loop = asyncio.new_event_loop() asyncio.set_event_loop(self.loop) - if sys.platform != 'win32': - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - watcher = asyncio.SafeChildWatcher() - watcher.attach_loop(self.loop) - asyncio.set_child_watcher(watcher) - def tearDown(self): try: - if sys.platform != 'win32': - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - asyncio.set_child_watcher(None) - super().tearDown() finally: self.loop.close() @@ -2871,8 +2988,10 @@ def tearDown(self): asyncio.get_running_loop = self.get_running_loop_saved asyncio.get_event_loop = self.get_event_loop_saved - if sys.platform != 'win32': + asyncio.Task = asyncio.tasks.Task = self._Task_saved + asyncio.Future = asyncio.futures.Future = self._Future_saved + if sys.platform != 'win32': def test_get_event_loop_new_process(self): # bpo-32126: The multiprocessing module used by # ProcessPoolExecutor is not functional when the @@ -2918,13 +3037,13 @@ def test_get_event_loop_returns_running_loop(self): class TestError(Exception): pass - class Policy(asyncio.DefaultEventLoopPolicy): + class Policy(test_utils.DefaultEventLoopPolicy): def get_event_loop(self): raise TestError - old_policy = asyncio.get_event_loop_policy() + old_policy = asyncio.events._get_event_loop_policy() try: - asyncio.set_event_loop_policy(Policy()) + asyncio.events._set_event_loop_policy(Policy()) loop = asyncio.new_event_loop() with self.assertRaises(TestError): @@ -2952,7 +3071,7 @@ async def func(): asyncio.get_event_loop() finally: - asyncio.set_event_loop_policy(old_policy) + asyncio.events._set_event_loop_policy(old_policy) if loop is not None: loop.close() @@ -2962,23 +3081,18 @@ async def func(): self.assertIs(asyncio._get_running_loop(), None) def test_get_event_loop_returns_running_loop2(self): - old_policy = asyncio.get_event_loop_policy() + old_policy = asyncio.events._get_event_loop_policy() try: - asyncio.set_event_loop_policy(asyncio.DefaultEventLoopPolicy()) + asyncio.events._set_event_loop_policy(test_utils.DefaultEventLoopPolicy()) loop = asyncio.new_event_loop() self.addCleanup(loop.close) - with self.assertWarns(DeprecationWarning) as cm: - loop2 = asyncio.get_event_loop() - self.addCleanup(loop2.close) - self.assertEqual(cm.filename, __file__) - asyncio.set_event_loop(None) with self.assertRaisesRegex(RuntimeError, 'no current'): asyncio.get_event_loop() - with self.assertRaisesRegex(RuntimeError, 'no running'): - asyncio.get_running_loop() - self.assertIs(asyncio._get_running_loop(), None) + asyncio.set_event_loop(None) + with self.assertRaisesRegex(RuntimeError, 'no current'): + asyncio.get_event_loop() async def func(): self.assertIs(asyncio.get_event_loop(), loop) @@ -2995,7 +3109,7 @@ async def func(): asyncio.get_event_loop() finally: - asyncio.set_event_loop_policy(old_policy) + asyncio.events._set_event_loop_policy(old_policy) if loop is not None: loop.close() @@ -3012,6 +3126,8 @@ class TestPyGetEventLoop(GetEventLoopTestsMixin, unittest.TestCase): get_running_loop_impl = events._py_get_running_loop get_event_loop_impl = events._py_get_event_loop + Task = asyncio.tasks._PyTask + Future = asyncio.futures._PyFuture try: import _asyncio # NoQA @@ -3026,6 +3142,8 @@ class TestCGetEventLoop(GetEventLoopTestsMixin, unittest.TestCase): get_running_loop_impl = events._c_get_running_loop get_event_loop_impl = events._c_get_event_loop + Task = asyncio.tasks._CTask + Future = asyncio.futures._CFuture class TestServer(unittest.TestCase): diff --git a/Lib/test/test_asyncio/test_free_threading.py b/Lib/test/test_asyncio/test_free_threading.py new file mode 100644 index 00000000000..c8de0d24499 --- /dev/null +++ b/Lib/test/test_asyncio/test_free_threading.py @@ -0,0 +1,235 @@ +import asyncio +import threading +import unittest +from threading import Thread +from unittest import TestCase +import weakref +from test import support +from test.support import threading_helper + +threading_helper.requires_working_threading(module=True) + + +class MyException(Exception): + pass + + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +class TestFreeThreading: + def test_all_tasks_race(self) -> None: + async def main(): + loop = asyncio.get_running_loop() + future = loop.create_future() + + async def coro(): + await future + + tasks = set() + + async with asyncio.TaskGroup() as tg: + for _ in range(100): + tasks.add(tg.create_task(coro())) + + all_tasks = asyncio.all_tasks(loop) + self.assertEqual(len(all_tasks), 101) + + for task in all_tasks: + self.assertEqual(task.get_loop(), loop) + self.assertFalse(task.done()) + + current = asyncio.current_task() + self.assertEqual(current.get_loop(), loop) + self.assertSetEqual(all_tasks, tasks | {current}) + future.set_result(None) + + def runner(): + with asyncio.Runner() as runner: + loop = runner.get_loop() + loop.set_task_factory(self.factory) + runner.run(main()) + + threads = [] + + for _ in range(10): + thread = Thread(target=runner) + threads.append(thread) + + with threading_helper.start_threads(threads): + pass + + def test_all_tasks_different_thread(self) -> None: + loop = None + started = threading.Event() + done = threading.Event() # used for main task not finishing early + async def coro(): + await asyncio.Future() + + lock = threading.Lock() + tasks = set() + + async def main(): + nonlocal tasks, loop + loop = asyncio.get_running_loop() + started.set() + for i in range(1000): + with lock: + asyncio.create_task(coro()) + tasks = asyncio.all_tasks(loop) + done.wait() + + runner = threading.Thread(target=lambda: asyncio.run(main())) + + def check(): + started.wait() + with lock: + self.assertSetEqual(tasks & asyncio.all_tasks(loop), tasks) + + threads = [threading.Thread(target=check) for _ in range(10)] + runner.start() + + with threading_helper.start_threads(threads): + pass + + done.set() + runner.join() + + def test_task_different_thread_finalized(self) -> None: + task = None + async def func(): + nonlocal task + task = asyncio.current_task() + def runner(): + with asyncio.Runner() as runner: + loop = runner.get_loop() + loop.set_task_factory(self.factory) + runner.run(func()) + thread = Thread(target=runner) + thread.start() + thread.join() + wr = weakref.ref(task) + del thread + del task + # task finalization in different thread shouldn't crash + support.gc_collect() + self.assertIsNone(wr()) + + def test_run_coroutine_threadsafe(self) -> None: + results = [] + + def in_thread(loop: asyncio.AbstractEventLoop): + coro = asyncio.sleep(0.1, result=42) + fut = asyncio.run_coroutine_threadsafe(coro, loop) + result = fut.result() + self.assertEqual(result, 42) + results.append(result) + + async def main(): + loop = asyncio.get_running_loop() + async with asyncio.TaskGroup() as tg: + for _ in range(10): + tg.create_task(asyncio.to_thread(in_thread, loop)) + self.assertEqual(results, [42] * 10) + + with asyncio.Runner() as r: + loop = r.get_loop() + loop.set_task_factory(self.factory) + r.run(main()) + + def test_run_coroutine_threadsafe_exception(self) -> None: + async def coro(): + await asyncio.sleep(0) + raise MyException("test") + + def in_thread(loop: asyncio.AbstractEventLoop): + fut = asyncio.run_coroutine_threadsafe(coro(), loop) + return fut.result() + + async def main(): + loop = asyncio.get_running_loop() + tasks = [] + for _ in range(10): + task = loop.create_task(asyncio.to_thread(in_thread, loop)) + tasks.append(task) + results = await asyncio.gather(*tasks, return_exceptions=True) + + self.assertEqual(len(results), 10) + for result in results: + self.assertIsInstance(result, MyException) + self.assertEqual(str(result), "test") + + with asyncio.Runner() as r: + loop = r.get_loop() + loop.set_task_factory(self.factory) + r.run(main()) + + +class TestPyFreeThreading(TestFreeThreading, TestCase): + + def setUp(self): + self._old_current_task = asyncio.current_task + asyncio.current_task = asyncio.tasks.current_task = asyncio.tasks._py_current_task + self._old_all_tasks = asyncio.all_tasks + asyncio.all_tasks = asyncio.tasks.all_tasks = asyncio.tasks._py_all_tasks + self._old_Task = asyncio.Task + asyncio.Task = asyncio.tasks.Task = asyncio.tasks._PyTask + self._old_Future = asyncio.Future + asyncio.Future = asyncio.futures.Future = asyncio.futures._PyFuture + return super().setUp() + + def tearDown(self): + asyncio.current_task = asyncio.tasks.current_task = self._old_current_task + asyncio.all_tasks = asyncio.tasks.all_tasks = self._old_all_tasks + asyncio.Task = asyncio.tasks.Task = self._old_Task + asyncio.Future = asyncio.tasks.Future = self._old_Future + return super().tearDown() + + def factory(self, loop, coro, **kwargs): + return asyncio.tasks._PyTask(coro, loop=loop, **kwargs) + + @unittest.skip("TODO: RUSTPYTHON; hangs - Python _current_tasks dict not thread-safe") + def test_all_tasks_race(self): + return super().test_all_tasks_race() + + +@unittest.skipUnless(hasattr(asyncio.tasks, "_c_all_tasks"), "requires _asyncio") +class TestCFreeThreading(TestFreeThreading, TestCase): + + def setUp(self): + self._old_current_task = asyncio.current_task + asyncio.current_task = asyncio.tasks.current_task = asyncio.tasks._c_current_task + self._old_all_tasks = asyncio.all_tasks + asyncio.all_tasks = asyncio.tasks.all_tasks = asyncio.tasks._c_all_tasks + self._old_Task = asyncio.Task + asyncio.Task = asyncio.tasks.Task = asyncio.tasks._CTask + self._old_Future = asyncio.Future + asyncio.Future = asyncio.futures.Future = asyncio.futures._CFuture + return super().setUp() + + def tearDown(self): + asyncio.current_task = asyncio.tasks.current_task = self._old_current_task + asyncio.all_tasks = asyncio.tasks.all_tasks = self._old_all_tasks + asyncio.Task = asyncio.tasks.Task = self._old_Task + asyncio.Future = asyncio.futures.Future = self._old_Future + return super().tearDown() + + + def factory(self, loop, coro, **kwargs): + return asyncio.tasks._CTask(coro, loop=loop, **kwargs) + + +class TestEagerPyFreeThreading(TestPyFreeThreading): + def factory(self, loop, coro, eager_start=True, **kwargs): + return asyncio.tasks._PyTask(coro, loop=loop, **kwargs, eager_start=eager_start) + + @unittest.skip("TODO: RUSTPYTHON; hangs - Python _current_tasks dict not thread-safe") + def test_all_tasks_race(self): + return super().test_all_tasks_race() + + +@unittest.skipUnless(hasattr(asyncio.tasks, "_c_all_tasks"), "requires _asyncio") +class TestEagerCFreeThreading(TestCFreeThreading, TestCase): + def factory(self, loop, coro, eager_start=True, **kwargs): + return asyncio.tasks._CTask(coro, loop=loop, **kwargs, eager_start=eager_start) diff --git a/Lib/test/test_asyncio/test_futures.py b/Lib/test/test_asyncio/test_futures.py index 571fbace020..54bf824fef7 100644 --- a/Lib/test/test_asyncio/test_futures.py +++ b/Lib/test/test_asyncio/test_futures.py @@ -17,7 +17,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) def _fakefunc(f): @@ -242,7 +242,7 @@ def test_uninitialized(self): def test_future_cancel_message_getter(self): f = self._new_future(loop=self.loop) - self.assertTrue(hasattr(f, '_cancel_message')) + self.assertHasAttr(f, '_cancel_message') self.assertEqual(f._cancel_message, None) f.cancel('my message') @@ -678,8 +678,6 @@ def __del__(self): fut = self._new_future(loop=self.loop) fut.set_result(Evil()) - # TODO: RUSTPYTHON - gc.get_referrers not implemented - @unittest.expectedFailure def test_future_cancelled_result_refcycles(self): f = self._new_future(loop=self.loop) f.cancel() @@ -691,8 +689,6 @@ def test_future_cancelled_result_refcycles(self): self.assertIsNotNone(exc) self.assertListEqual(gc.get_referrers(exc), []) - # TODO: RUSTPYTHON - gc.get_referrers not implemented - @unittest.expectedFailure def test_future_cancelled_exception_refcycles(self): f = self._new_future(loop=self.loop) f.cancel() @@ -720,16 +716,6 @@ def test_future_del_segfault(self): with self.assertRaises(AttributeError): del fut._log_traceback - # TODO: RUSTPYTHON - gc.get_referents not implemented - @unittest.expectedFailure - def test_future_iter_get_referents_segfault(self): - # See https://github.com/python/cpython/issues/122695 - import _asyncio - it = iter(self._new_future(loop=self.loop)) - del it - evil = gc.get_referents(_asyncio) - gc.collect() - def test_callbacks_copy(self): # See https://github.com/python/cpython/issues/125789 # In C implementation, the `_callbacks` attribute @@ -748,6 +734,10 @@ def test_callbacks_copy(self): fut.remove_done_callback(f2) self.assertIsNone(fut._callbacks) + @unittest.expectedFailure # TODO: RUSTPYTHON; - gc.get_referents not implemented + def test_future_iter_get_referents_segfault(self): + return super().test_future_iter_get_referents_segfault() + @unittest.skipUnless(hasattr(futures, '_CFuture'), 'requires the C _asyncio module') diff --git a/Lib/test/test_asyncio/test_futures2.py b/Lib/test/test_asyncio/test_futures2.py index b7cfffb76bd..c7c0ebdac1b 100644 --- a/Lib/test/test_asyncio/test_futures2.py +++ b/Lib/test/test_asyncio/test_futures2.py @@ -7,7 +7,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class FutureTests: diff --git a/Lib/test/test_asyncio/test_graph.py b/Lib/test/test_asyncio/test_graph.py new file mode 100644 index 00000000000..2f22fbccba4 --- /dev/null +++ b/Lib/test/test_asyncio/test_graph.py @@ -0,0 +1,445 @@ +import asyncio +import io +import unittest + + +# To prevent a warning "test altered the execution environment" +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +def capture_test_stack(*, fut=None, depth=1): + + def walk(s): + ret = [ + (f"T<{n}>" if '-' not in (n := s.future.get_name()) else 'T') + if isinstance(s.future, asyncio.Task) else 'F' + ] + + ret.append( + [ + ( + f"s {entry.frame.f_code.co_name}" + if entry.frame.f_generator is None else + ( + f"a {entry.frame.f_generator.cr_code.co_name}" + if hasattr(entry.frame.f_generator, 'cr_code') else + f"ag {entry.frame.f_generator.ag_code.co_name}" + ) + ) for entry in s.call_stack + ] + ) + + ret.append( + sorted([ + walk(ab) for ab in s.awaited_by + ], key=lambda entry: entry[0]) + ) + + return ret + + buf = io.StringIO() + asyncio.print_call_graph(fut, file=buf, depth=depth+1) + + stack = asyncio.capture_call_graph(fut, depth=depth) + return walk(stack), buf.getvalue() + + +class CallStackTestBase: + + async def test_stack_tgroup(self): + + stack_for_c5 = None + + def c5(): + nonlocal stack_for_c5 + stack_for_c5 = capture_test_stack(depth=2) + + async def c4(): + await asyncio.sleep(0) + c5() + + async def c3(): + await c4() + + async def c2(): + await c3() + + async def c1(task): + await task + + async def main(): + async with asyncio.TaskGroup() as tg: + task = tg.create_task(c2(), name="c2_root") + tg.create_task(c1(task), name="sub_main_1") + tg.create_task(c1(task), name="sub_main_2") + + await main() + + self.assertEqual(stack_for_c5[0], [ + # task name + 'T', + # call stack + ['s c5', 'a c4', 'a c3', 'a c2'], + # awaited by + [ + ['T', + ['a _aexit', 'a __aexit__', 'a main', 'a test_stack_tgroup'], [] + ], + ['T', + ['a c1'], + [ + ['T', + ['a _aexit', 'a __aexit__', 'a main', 'a test_stack_tgroup'], [] + ] + ] + ], + ['T', + ['a c1'], + [ + ['T', + ['a _aexit', 'a __aexit__', 'a main', 'a test_stack_tgroup'], [] + ] + ] + ] + ] + ]) + + self.assertIn( + ' async CallStackTestBase.test_stack_tgroup()', + stack_for_c5[1]) + + + async def test_stack_async_gen(self): + + stack_for_gen_nested_call = None + + async def gen_nested_call(): + nonlocal stack_for_gen_nested_call + stack_for_gen_nested_call = capture_test_stack() + + async def gen(): + for num in range(2): + yield num + if num == 1: + await gen_nested_call() + + async def main(): + async for el in gen(): + pass + + await main() + + self.assertEqual(stack_for_gen_nested_call[0], [ + 'T', + [ + 's capture_test_stack', + 'a gen_nested_call', + 'ag gen', + 'a main', + 'a test_stack_async_gen' + ], + [] + ]) + + self.assertIn( + 'async generator CallStackTestBase.test_stack_async_gen..gen()', + stack_for_gen_nested_call[1]) + + async def test_stack_gather(self): + + stack_for_deep = None + + async def deep(): + await asyncio.sleep(0) + nonlocal stack_for_deep + stack_for_deep = capture_test_stack() + + async def c1(): + await asyncio.sleep(0) + await deep() + + async def c2(): + await asyncio.sleep(0) + + async def main(): + await asyncio.gather(c1(), c2()) + + await main() + + self.assertEqual(stack_for_deep[0], [ + 'T', + ['s capture_test_stack', 'a deep', 'a c1'], + [ + ['T', ['a main', 'a test_stack_gather'], []] + ] + ]) + + async def test_stack_shield(self): + + stack_for_shield = None + + async def deep(): + await asyncio.sleep(0) + nonlocal stack_for_shield + stack_for_shield = capture_test_stack() + + async def c1(): + await asyncio.sleep(0) + await deep() + + async def main(): + await asyncio.shield(c1()) + + await main() + + self.assertEqual(stack_for_shield[0], [ + 'T', + ['s capture_test_stack', 'a deep', 'a c1'], + [ + ['T', ['a main', 'a test_stack_shield'], []] + ] + ]) + + async def test_stack_timeout(self): + + stack_for_inner = None + + async def inner(): + await asyncio.sleep(0) + nonlocal stack_for_inner + stack_for_inner = capture_test_stack() + + async def c1(): + async with asyncio.timeout(1): + await asyncio.sleep(0) + await inner() + + async def main(): + await asyncio.shield(c1()) + + await main() + + self.assertEqual(stack_for_inner[0], [ + 'T', + ['s capture_test_stack', 'a inner', 'a c1'], + [ + ['T', ['a main', 'a test_stack_timeout'], []] + ] + ]) + + async def test_stack_wait(self): + + stack_for_inner = None + + async def inner(): + await asyncio.sleep(0) + nonlocal stack_for_inner + stack_for_inner = capture_test_stack() + + async def c1(): + async with asyncio.timeout(1): + await asyncio.sleep(0) + await inner() + + async def c2(): + for i in range(3): + await asyncio.sleep(0) + + async def main(t1, t2): + while True: + _, pending = await asyncio.wait([t1, t2]) + if not pending: + break + + t1 = asyncio.create_task(c1()) + t2 = asyncio.create_task(c2()) + try: + await main(t1, t2) + finally: + await t1 + await t2 + + self.assertEqual(stack_for_inner[0], [ + 'T', + ['s capture_test_stack', 'a inner', 'a c1'], + [ + ['T', + ['a _wait', 'a wait', 'a main', 'a test_stack_wait'], + [] + ] + ] + ]) + + async def test_stack_task(self): + + stack_for_inner = None + + async def inner(): + await asyncio.sleep(0) + nonlocal stack_for_inner + stack_for_inner = capture_test_stack() + + async def c1(): + await inner() + + async def c2(): + await asyncio.create_task(c1(), name='there there') + + async def main(): + await c2() + + await main() + + self.assertEqual(stack_for_inner[0], [ + 'T', + ['s capture_test_stack', 'a inner', 'a c1'], + [['T', ['a c2', 'a main', 'a test_stack_task'], []]] + ]) + + async def test_stack_future(self): + + stack_for_fut = None + + async def a2(fut): + await fut + + async def a1(fut): + await a2(fut) + + async def b1(fut): + await fut + + async def main(): + nonlocal stack_for_fut + + fut = asyncio.Future() + async with asyncio.TaskGroup() as g: + g.create_task(a1(fut), name="task A") + g.create_task(b1(fut), name='task B') + + for _ in range(5): + # Do a few iterations to ensure that both a1 and b1 + # await on the future + await asyncio.sleep(0) + + stack_for_fut = capture_test_stack(fut=fut) + fut.set_result(None) + + await main() + + self.assertEqual(stack_for_fut[0], + ['F', + [], + [ + ['T', + ['a a2', 'a a1'], + [['T', ['a test_stack_future'], []]] + ], + ['T', + ['a b1'], + [['T', ['a test_stack_future'], []]] + ], + ]] + ) + + self.assertTrue(stack_for_fut[1].startswith('* Future(id=')) + + +@unittest.skipIf( + not hasattr(asyncio.futures, "_c_future_add_to_awaited_by"), + "C-accelerated asyncio call graph backend missing", +) +class TestCallStackC(CallStackTestBase, unittest.IsolatedAsyncioTestCase): + def setUp(self): + futures = asyncio.futures + tasks = asyncio.tasks + + self._Future = asyncio.Future + asyncio.Future = futures.Future = futures._CFuture + + self._Task = asyncio.Task + asyncio.Task = tasks.Task = tasks._CTask + + self._future_add_to_awaited_by = asyncio.future_add_to_awaited_by + futures.future_add_to_awaited_by = futures._c_future_add_to_awaited_by + asyncio.future_add_to_awaited_by = futures.future_add_to_awaited_by + + self._future_discard_from_awaited_by = asyncio.future_discard_from_awaited_by + futures.future_discard_from_awaited_by = futures._c_future_discard_from_awaited_by + asyncio.future_discard_from_awaited_by = futures.future_discard_from_awaited_by + + self._current_task = asyncio.current_task + asyncio.current_task = asyncio.tasks.current_task = tasks._c_current_task + + def tearDown(self): + futures = asyncio.futures + tasks = asyncio.tasks + + futures.future_discard_from_awaited_by = self._future_discard_from_awaited_by + asyncio.future_discard_from_awaited_by = self._future_discard_from_awaited_by + del self._future_discard_from_awaited_by + + futures.future_add_to_awaited_by = self._future_add_to_awaited_by + asyncio.future_add_to_awaited_by = self._future_add_to_awaited_by + del self._future_add_to_awaited_by + + asyncio.Task = self._Task + tasks.Task = self._Task + del self._Task + + asyncio.Future = self._Future + futures.Future = self._Future + del self._Future + + asyncio.current_task = asyncio.tasks.current_task = self._current_task + + +@unittest.skipIf( + not hasattr(asyncio.futures, "_py_future_add_to_awaited_by"), + "Pure Python asyncio call graph backend missing", +) +class TestCallStackPy(CallStackTestBase, unittest.IsolatedAsyncioTestCase): + def setUp(self): + futures = asyncio.futures + tasks = asyncio.tasks + + self._Future = asyncio.Future + asyncio.Future = futures.Future = futures._PyFuture + + self._Task = asyncio.Task + asyncio.Task = tasks.Task = tasks._PyTask + + self._future_add_to_awaited_by = asyncio.future_add_to_awaited_by + futures.future_add_to_awaited_by = futures._py_future_add_to_awaited_by + asyncio.future_add_to_awaited_by = futures.future_add_to_awaited_by + + self._future_discard_from_awaited_by = asyncio.future_discard_from_awaited_by + futures.future_discard_from_awaited_by = futures._py_future_discard_from_awaited_by + asyncio.future_discard_from_awaited_by = futures.future_discard_from_awaited_by + + self._current_task = asyncio.current_task + asyncio.current_task = asyncio.tasks.current_task = tasks._py_current_task + + + def tearDown(self): + futures = asyncio.futures + tasks = asyncio.tasks + + futures.future_discard_from_awaited_by = self._future_discard_from_awaited_by + asyncio.future_discard_from_awaited_by = self._future_discard_from_awaited_by + del self._future_discard_from_awaited_by + + futures.future_add_to_awaited_by = self._future_add_to_awaited_by + asyncio.future_add_to_awaited_by = self._future_add_to_awaited_by + del self._future_add_to_awaited_by + + asyncio.Task = self._Task + tasks.Task = self._Task + del self._Task + + asyncio.Future = self._Future + futures.Future = self._Future + del self._Future + + asyncio.current_task = asyncio.tasks.current_task = self._current_task diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index a0884bffe6b..e025d2990a3 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -14,24 +14,24 @@ r'(, value:\d)?' r'(, waiters:\d+)?' r'(, waiters:\d+\/\d+)?' # barrier - r')\]>\Z' + r')\]>\z' ) RGX_REPR = re.compile(STR_RGX_REPR) def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class LockTests(unittest.IsolatedAsyncioTestCase): async def test_repr(self): lock = asyncio.Lock() - self.assertTrue(repr(lock).endswith('[unlocked]>')) + self.assertEndsWith(repr(lock), '[unlocked]>') self.assertTrue(RGX_REPR.match(repr(lock))) await lock.acquire() - self.assertTrue(repr(lock).endswith('[locked]>')) + self.assertEndsWith(repr(lock), '[locked]>') self.assertTrue(RGX_REPR.match(repr(lock))) async def test_lock(self): @@ -39,7 +39,7 @@ async def test_lock(self): with self.assertRaisesRegex( TypeError, - "object Lock can't be used in 'await' expression" + "'Lock' object can't be awaited" ): await lock @@ -77,7 +77,7 @@ async def test_lock_by_with_statement(self): self.assertFalse(lock.locked()) with self.assertRaisesRegex( TypeError, - r"object \w+ can't be used in 'await' expression" + r"'\w+' object can't be awaited" ): with await lock: pass @@ -286,12 +286,12 @@ class EventTests(unittest.IsolatedAsyncioTestCase): def test_repr(self): ev = asyncio.Event() - self.assertTrue(repr(ev).endswith('[unset]>')) + self.assertEndsWith(repr(ev), '[unset]>') match = RGX_REPR.match(repr(ev)) self.assertEqual(match.group('extras'), 'unset') ev.set() - self.assertTrue(repr(ev).endswith('[set]>')) + self.assertEndsWith(repr(ev), '[set]>') self.assertTrue(RGX_REPR.match(repr(ev))) ev._waiters.append(mock.Mock()) @@ -916,11 +916,11 @@ def test_initial_value_zero(self): async def test_repr(self): sem = asyncio.Semaphore() - self.assertTrue(repr(sem).endswith('[unlocked, value:1]>')) + self.assertEndsWith(repr(sem), '[unlocked, value:1]>') self.assertTrue(RGX_REPR.match(repr(sem))) await sem.acquire() - self.assertTrue(repr(sem).endswith('[locked]>')) + self.assertEndsWith(repr(sem), '[locked]>') self.assertTrue('waiters' not in repr(sem)) self.assertTrue(RGX_REPR.match(repr(sem))) @@ -941,7 +941,7 @@ async def test_semaphore(self): with self.assertRaisesRegex( TypeError, - "object Semaphore can't be used in 'await' expression", + "'Semaphore' object can't be awaited", ): await sem @@ -1194,14 +1194,14 @@ async def c3(result): self.assertEqual([2, 3], result) async def test_acquire_fifo_order_4(self): - # Test that a successfule `acquire()` will wake up multiple Tasks + # Test that a successful `acquire()` will wake up multiple Tasks # that were waiting in the Semaphore queue due to FIFO rules. sem = asyncio.Semaphore(0) result = [] count = 0 async def c1(result): - # First task immediatlly waits for semaphore. It will be awoken by c2. + # First task immediately waits for semaphore. It will be awoken by c2. self.assertEqual(sem._value, 0) await sem.acquire() # We should have woken up all waiting tasks now. @@ -1270,7 +1270,7 @@ async def test_barrier(self): self.assertIn("filling", repr(barrier)) with self.assertRaisesRegex( TypeError, - "object Barrier can't be used in 'await' expression", + "'Barrier' object can't be awaited", ): await barrier @@ -1475,13 +1475,13 @@ async def coro(): # first time waiting await barrier.wait() - # after wainting once for all tasks + # after waiting once for all tasks if rewait_n > 0: rewait_n -= 1 # wait again only for rewait tasks await barrier.wait() else: - # wait for end of draining state` + # wait for end of draining state await barrier_nowaiting.wait() # wait for other waiting tasks await barrier.wait() @@ -1780,7 +1780,7 @@ async def coro(): self.assertEqual(barrier.n_waiting, 0) async def test_abort_barrier_when_exception_then_resetting(self): - # test from threading.Barrier: see `lock_tests.test_abort_and_reset`` + # test from threading.Barrier: see `lock_tests.test_abort_and_reset` barrier1 = asyncio.Barrier(self.N) barrier2 = asyncio.Barrier(self.N) results1 = [] diff --git a/Lib/test/test_asyncio/test_pep492.py b/Lib/test/test_asyncio/test_pep492.py index dc25a46985e..a0c8434c945 100644 --- a/Lib/test/test_asyncio/test_pep492.py +++ b/Lib/test/test_asyncio/test_pep492.py @@ -11,7 +11,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) # Test that asyncio.iscoroutine() uses collections.abc.Coroutine @@ -77,7 +77,7 @@ async def test(lock): self.assertFalse(lock.locked()) with self.assertRaisesRegex( TypeError, - "can't be used in 'await' expression" + "can't be awaited" ): with await lock: pass @@ -124,10 +124,10 @@ def foo(): yield self.assertFalse(asyncio.iscoroutine(foo())) - def test_iscoroutinefunction(self): async def foo(): pass - self.assertTrue(asyncio.iscoroutinefunction(foo)) + with self.assertWarns(DeprecationWarning): + self.assertTrue(asyncio.iscoroutinefunction(foo)) def test_async_def_coroutines(self): async def bar(): diff --git a/Lib/test/test_asyncio/test_proactor_events.py b/Lib/test/test_asyncio/test_proactor_events.py index 40cb5213c63..edfad5e11db 100644 --- a/Lib/test/test_asyncio/test_proactor_events.py +++ b/Lib/test/test_asyncio/test_proactor_events.py @@ -18,7 +18,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) def close_transport(transport): diff --git a/Lib/test/test_asyncio/test_protocols.py b/Lib/test/test_asyncio/test_protocols.py index 0f232631867..29d3bd22705 100644 --- a/Lib/test/test_asyncio/test_protocols.py +++ b/Lib/test/test_asyncio/test_protocols.py @@ -7,7 +7,7 @@ def tearDownModule(): # not needed for the test file but added for uniformness with all other # asyncio test files for the sake of unified cleanup - asyncio.set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class ProtocolsAbsTests(unittest.TestCase): @@ -19,7 +19,7 @@ def test_base_protocol(self): self.assertIsNone(p.connection_lost(f)) self.assertIsNone(p.pause_writing()) self.assertIsNone(p.resume_writing()) - self.assertFalse(hasattr(p, '__dict__')) + self.assertNotHasAttr(p, '__dict__') def test_protocol(self): f = mock.Mock() @@ -30,7 +30,7 @@ def test_protocol(self): self.assertIsNone(p.eof_received()) self.assertIsNone(p.pause_writing()) self.assertIsNone(p.resume_writing()) - self.assertFalse(hasattr(p, '__dict__')) + self.assertNotHasAttr(p, '__dict__') def test_buffered_protocol(self): f = mock.Mock() @@ -41,7 +41,7 @@ def test_buffered_protocol(self): self.assertIsNone(p.buffer_updated(150)) self.assertIsNone(p.pause_writing()) self.assertIsNone(p.resume_writing()) - self.assertFalse(hasattr(p, '__dict__')) + self.assertNotHasAttr(p, '__dict__') def test_datagram_protocol(self): f = mock.Mock() @@ -50,7 +50,7 @@ def test_datagram_protocol(self): self.assertIsNone(dp.connection_lost(f)) self.assertIsNone(dp.error_received(f)) self.assertIsNone(dp.datagram_received(f, f)) - self.assertFalse(hasattr(dp, '__dict__')) + self.assertNotHasAttr(dp, '__dict__') def test_subprocess_protocol(self): f = mock.Mock() @@ -60,7 +60,7 @@ def test_subprocess_protocol(self): self.assertIsNone(sp.pipe_data_received(1, f)) self.assertIsNone(sp.pipe_connection_lost(1, f)) self.assertIsNone(sp.process_exited()) - self.assertFalse(hasattr(sp, '__dict__')) + self.assertNotHasAttr(sp, '__dict__') if __name__ == '__main__': diff --git a/Lib/test/test_asyncio/test_queues.py b/Lib/test/test_asyncio/test_queues.py index 5019e9a2935..54bbe79f81f 100644 --- a/Lib/test/test_asyncio/test_queues.py +++ b/Lib/test/test_asyncio/test_queues.py @@ -6,7 +6,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class QueueBasicTests(unittest.IsolatedAsyncioTestCase): @@ -18,7 +18,7 @@ async def _test_repr_or_str(self, fn, expect_id): appear in fn(Queue()). """ q = asyncio.Queue() - self.assertTrue(fn(q).startswith('= self._max_clients: diff --git a/Lib/test/test_asyncio/test_sslproto.py b/Lib/test/test_asyncio/test_sslproto.py index ed9e0b344ec..7ab6e1511d7 100644 --- a/Lib/test/test_asyncio/test_sslproto.py +++ b/Lib/test/test_asyncio/test_sslproto.py @@ -21,7 +21,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) @unittest.skipIf(ssl is None, 'No ssl module') @@ -282,8 +282,7 @@ def buffer_updated(self, nsize): with self.assertRaisesRegex(RuntimeError, 'empty buffer'): protocols._feed_data_to_buffered_proto(proto, b'12345') - # TODO: RUSTPYTHON - gc.collect() doesn't release SSLContext properly - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; - gc.collect() doesn't release SSLContext properly def test_start_tls_client_reg_proto_1(self): HELLO_MSG = b'1' * self.PAYLOAD_SIZE @@ -350,8 +349,7 @@ async def client(addr): support.gc_collect() self.assertIsNone(client_context()) - # TODO: RUSTPYTHON - gc.collect() doesn't release SSLContext properly - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; - gc.collect() doesn't release SSLContext properly def test_create_connection_memory_leak(self): HELLO_MSG = b'1' * self.PAYLOAD_SIZE @@ -670,8 +668,7 @@ async def main(): self.loop.run_until_complete(main()) - # TODO: RUSTPYTHON - gc.collect() doesn't release SSLContext properly - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; - gc.collect() doesn't release SSLContext properly def test_handshake_timeout(self): # bpo-29970: Check that a connection is aborted if handshake is not # completed in timeout period, instead of remaining open indefinitely diff --git a/Lib/test/test_asyncio/test_staggered.py b/Lib/test/test_asyncio/test_staggered.py index 40455a3804e..32e4817b70d 100644 --- a/Lib/test/test_asyncio/test_staggered.py +++ b/Lib/test/test_asyncio/test_staggered.py @@ -8,7 +8,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class StaggeredTests(unittest.IsolatedAsyncioTestCase): diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py index dcf8bc79a2f..5f0fc6a7a9d 100644 --- a/Lib/test/test_asyncio/test_streams.py +++ b/Lib/test/test_asyncio/test_streams.py @@ -1,15 +1,12 @@ """Tests for streams.py.""" import gc -import os import queue import pickle import socket -import sys import threading import unittest from unittest import mock -import warnings try: import ssl except ImportError: @@ -17,11 +14,11 @@ import asyncio from test.test_asyncio import utils as test_utils -from test.support import requires_subprocess, socket_helper +from test.support import socket_helper def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class StreamTests(test_utils.TestCase): @@ -50,7 +47,7 @@ def _basetest_open_connection(self, open_connection_fut): self.assertEqual(data, b'HTTP/1.0 200 OK\r\n') f = reader.read() data = self.loop.run_until_complete(f) - self.assertTrue(data.endswith(b'\r\n\r\nTest message')) + self.assertEndsWith(data, b'\r\n\r\nTest message') writer.close() self.assertEqual(messages, []) @@ -75,7 +72,7 @@ def _basetest_open_connection_no_loop_ssl(self, open_connection_fut): writer.write(b'GET / HTTP/1.0\r\n\r\n') f = reader.read() data = self.loop.run_until_complete(f) - self.assertTrue(data.endswith(b'\r\n\r\nTest message')) + self.assertEndsWith(data, b'\r\n\r\nTest message') writer.close() self.assertEqual(messages, []) @@ -822,52 +819,6 @@ async def client(addr): self.assertEqual(msg1, b"hello world 1!\n") self.assertEqual(msg2, b"hello world 2!\n") - @unittest.skipIf(sys.platform == 'win32', "Don't have pipes") - @requires_subprocess() - def test_read_all_from_pipe_reader(self): - # See asyncio issue 168. This test is derived from the example - # subprocess_attach_read_pipe.py, but we configure the - # StreamReader's limit so that twice it is less than the size - # of the data writer. Also we must explicitly attach a child - # watcher to the event loop. - - code = """\ -import os, sys -fd = int(sys.argv[1]) -os.write(fd, b'data') -os.close(fd) -""" - rfd, wfd = os.pipe() - args = [sys.executable, '-c', code, str(wfd)] - - pipe = open(rfd, 'rb', 0) - reader = asyncio.StreamReader(loop=self.loop, limit=1) - protocol = asyncio.StreamReaderProtocol(reader, loop=self.loop) - transport, _ = self.loop.run_until_complete( - self.loop.connect_read_pipe(lambda: protocol, pipe)) - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - watcher = asyncio.SafeChildWatcher() - watcher.attach_loop(self.loop) - try: - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - asyncio.set_child_watcher(watcher) - create = asyncio.create_subprocess_exec( - *args, - pass_fds={wfd}, - ) - proc = self.loop.run_until_complete(create) - self.loop.run_until_complete(proc.wait()) - finally: - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - asyncio.set_child_watcher(None) - - os.close(wfd) - data = self.loop.run_until_complete(reader.read(-1)) - self.assertEqual(data, b'data') - def test_streamreader_constructor_without_loop(self): with self.assertRaisesRegex(RuntimeError, 'no current event loop'): asyncio.StreamReader() @@ -1048,7 +999,7 @@ def test_wait_closed_on_close(self): self.assertEqual(data, b'HTTP/1.0 200 OK\r\n') f = rd.read() data = self.loop.run_until_complete(f) - self.assertTrue(data.endswith(b'\r\n\r\nTest message')) + self.assertEndsWith(data, b'\r\n\r\nTest message') self.assertFalse(wr.is_closing()) wr.close() self.assertTrue(wr.is_closing()) @@ -1074,7 +1025,7 @@ async def inner(httpd): data = await rd.readline() self.assertEqual(data, b'HTTP/1.0 200 OK\r\n') data = await rd.read() - self.assertTrue(data.endswith(b'\r\n\r\nTest message')) + self.assertEndsWith(data, b'\r\n\r\nTest message') wr.close() await wr.wait_closed() @@ -1094,7 +1045,7 @@ async def inner(httpd): data = await rd.readline() self.assertEqual(data, b'HTTP/1.0 200 OK\r\n') data = await rd.read() - self.assertTrue(data.endswith(b'\r\n\r\nTest message')) + self.assertEndsWith(data, b'\r\n\r\nTest message') wr.close() with self.assertRaises(ConnectionResetError): wr.write(b'data') @@ -1135,12 +1086,12 @@ async def inner(httpd): data = await rd.readline() self.assertEqual(data, b'HTTP/1.0 200 OK\r\n') data = await rd.read() - self.assertTrue(data.endswith(b'\r\n\r\nTest message')) + self.assertEndsWith(data, b'\r\n\r\nTest message') with self.assertWarns(ResourceWarning) as cm: del wr gc.collect() self.assertEqual(len(cm.warnings), 1) - self.assertTrue(str(cm.warnings[0].message).startswith("unclosed None: def test_subprocess_protocol_events(self): # gh-108973: Test that all subprocess protocol methods are called. - # The protocol methods are not called in a determistic order. + # The protocol methods are not called in a deterministic order. # The order depends on the event loop and the operating system. events = [] fds = [1, 2] @@ -931,55 +922,31 @@ async def main(): # Unix class SubprocessWatcherMixin(SubprocessMixin): - Watcher = None - def setUp(self): super().setUp() - policy = asyncio.get_event_loop_policy() - self.loop = policy.new_event_loop() + self.loop = asyncio.new_event_loop() self.set_event_loop(self.loop) - watcher = self._get_watcher() - watcher.attach_loop(self.loop) - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - policy.set_child_watcher(watcher) + def test_watcher_implementation(self): + loop = self.loop + watcher = loop._watcher + if unix_events.can_use_pidfd(): + self.assertIsInstance(watcher, unix_events._PidfdChildWatcher) + else: + self.assertIsInstance(watcher, unix_events._ThreadedChildWatcher) - def tearDown(self): - super().tearDown() - policy = asyncio.get_event_loop_policy() - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - watcher = policy.get_child_watcher() - policy.set_child_watcher(None) - watcher.attach_loop(None) - watcher.close() class SubprocessThreadedWatcherTests(SubprocessWatcherMixin, test_utils.TestCase): + def setUp(self): + self._original_can_use_pidfd = unix_events.can_use_pidfd + # Force the use of the threaded child watcher + unix_events.can_use_pidfd = mock.Mock(return_value=False) + super().setUp() - def _get_watcher(self): - return unix_events.ThreadedChildWatcher() - - class SubprocessSafeWatcherTests(SubprocessWatcherMixin, - test_utils.TestCase): - - def _get_watcher(self): - with self.assertWarns(DeprecationWarning): - return unix_events.SafeChildWatcher() - - class MultiLoopChildWatcherTests(test_utils.TestCase): - - def test_warns(self): - with self.assertWarns(DeprecationWarning): - unix_events.MultiLoopChildWatcher() - - class SubprocessFastWatcherTests(SubprocessWatcherMixin, - test_utils.TestCase): - - def _get_watcher(self): - with self.assertWarns(DeprecationWarning): - return unix_events.FastChildWatcher() + def tearDown(self): + unix_events.can_use_pidfd = self._original_can_use_pidfd + return super().tearDown() @unittest.skipUnless( unix_events.can_use_pidfd(), @@ -988,70 +955,8 @@ def _get_watcher(self): class SubprocessPidfdWatcherTests(SubprocessWatcherMixin, test_utils.TestCase): - def _get_watcher(self): - return unix_events.PidfdChildWatcher() + pass - - class GenericWatcherTests(test_utils.TestCase): - - def test_create_subprocess_fails_with_inactive_watcher(self): - watcher = mock.create_autospec(asyncio.AbstractChildWatcher) - watcher.is_active.return_value = False - - async def execute(): - asyncio.set_child_watcher(watcher) - - with self.assertRaises(RuntimeError): - await subprocess.create_subprocess_exec( - os_helper.FakePath(sys.executable), '-c', 'pass') - - watcher.add_child_handler.assert_not_called() - - with asyncio.Runner(loop_factory=asyncio.new_event_loop) as runner: - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - self.assertIsNone(runner.run(execute())) - self.assertListEqual(watcher.mock_calls, [ - mock.call.__enter__(), - mock.call.is_active(), - mock.call.__exit__(RuntimeError, mock.ANY, mock.ANY), - ], watcher.mock_calls) - - - @unittest.skipUnless( - unix_events.can_use_pidfd(), - "operating system does not support pidfds", - ) - def test_create_subprocess_with_pidfd(self): - async def in_thread(): - proc = await asyncio.create_subprocess_exec( - *PROGRAM_CAT, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - ) - stdout, stderr = await proc.communicate(b"some data") - return proc.returncode, stdout - - async def main(): - # asyncio.Runner did not call asyncio.set_event_loop() - with warnings.catch_warnings(): - warnings.simplefilter('error', DeprecationWarning) - # get_event_loop() raises DeprecationWarning if - # set_event_loop() was never called and RuntimeError if - # it was called at least once. - with self.assertRaises((RuntimeError, DeprecationWarning)): - asyncio.get_event_loop_policy().get_event_loop() - return await asyncio.to_thread(asyncio.run, in_thread()) - with self.assertWarns(DeprecationWarning): - asyncio.set_child_watcher(asyncio.PidfdChildWatcher()) - try: - with asyncio.Runner(loop_factory=asyncio.new_event_loop) as runner: - returncode, stdout = runner.run(main()) - self.assertEqual(returncode, 0) - self.assertEqual(stdout, b'some data') - finally: - with self.assertWarns(DeprecationWarning): - asyncio.set_child_watcher(None) else: # Windows class SubprocessProactorTests(SubprocessMixin, test_utils.TestCase): diff --git a/Lib/test/test_asyncio/test_taskgroups.py b/Lib/test/test_asyncio/test_taskgroups.py index 5910d616610..d4b2554dda9 100644 --- a/Lib/test/test_asyncio/test_taskgroups.py +++ b/Lib/test/test_asyncio/test_taskgroups.py @@ -15,7 +15,7 @@ # To prevent a warning "test altered the execution environment" def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class MyExc(Exception): @@ -30,6 +30,15 @@ def get_error_types(eg): return {type(exc) for exc in eg.exceptions} +def no_other_refs(): + # due to gh-124392 coroutines now refer to their locals + coro = asyncio.current_task().get_coro() + frame = sys._getframe(1) + while coro.cr_frame != frame: + coro = coro.cr_await + return [coro] + + def set_gc_state(enabled): was_enabled = gc.isenabled() if enabled: @@ -918,7 +927,6 @@ async def outer(): await outer() - @unittest.skip("RUSTPYTHON: gc.get_referrers not implemented") async def test_exception_refcycles_direct(self): """Test that TaskGroup doesn't keep a reference to the raised ExceptionGroup""" tg = asyncio.TaskGroup() @@ -934,10 +942,9 @@ class _Done(Exception): exc = e self.assertIsNotNone(exc) - self.assertListEqual(gc.get_referrers(exc), []) + self.assertListEqual(gc.get_referrers(exc), no_other_refs()) - @unittest.skip("RUSTPYTHON: gc.get_referrers not implemented") async def test_exception_refcycles_errors(self): """Test that TaskGroup deletes self._errors, and __aexit__ args""" tg = asyncio.TaskGroup() @@ -953,10 +960,9 @@ class _Done(Exception): exc = excs.exceptions[0] self.assertIsInstance(exc, _Done) - self.assertListEqual(gc.get_referrers(exc), []) + self.assertListEqual(gc.get_referrers(exc), no_other_refs()) - @unittest.skip("RUSTPYTHON: gc.get_referrers not implemented") async def test_exception_refcycles_parent_task(self): """Test that TaskGroup deletes self._parent_task""" tg = asyncio.TaskGroup() @@ -976,10 +982,9 @@ async def coro_fn(): exc = excs.exceptions[0].exceptions[0] self.assertIsInstance(exc, _Done) - self.assertListEqual(gc.get_referrers(exc), []) + self.assertListEqual(gc.get_referrers(exc), no_other_refs()) - @unittest.skip("RUSTPYTHON: gc.get_referrers not implemented") async def test_exception_refcycles_parent_task_wr(self): """Test that TaskGroup deletes self._parent_task and create_task() deletes task""" tg = asyncio.TaskGroup() @@ -1001,9 +1006,8 @@ async def coro_fn(): self.assertIsNone(task_wr()) self.assertIsInstance(exc, _Done) - self.assertListEqual(gc.get_referrers(exc), []) + self.assertListEqual(gc.get_referrers(exc), no_other_refs()) - @unittest.skip("RUSTPYTHON: gc.get_referrers not implemented") async def test_exception_refcycles_propagate_cancellation_error(self): """Test that TaskGroup deletes propagate_cancellation_error""" tg = asyncio.TaskGroup() @@ -1017,9 +1021,8 @@ async def test_exception_refcycles_propagate_cancellation_error(self): exc = e.__cause__ self.assertIsInstance(exc, asyncio.CancelledError) - self.assertListEqual(gc.get_referrers(exc), []) + self.assertListEqual(gc.get_referrers(exc), no_other_refs()) - @unittest.skip("RUSTPYTHON: gc.get_referrers not implemented") async def test_exception_refcycles_base_error(self): """Test that TaskGroup deletes self._base_error""" class MyKeyboardInterrupt(KeyboardInterrupt): @@ -1035,7 +1038,19 @@ class MyKeyboardInterrupt(KeyboardInterrupt): exc = e self.assertIsNotNone(exc) - self.assertListEqual(gc.get_referrers(exc), []) + self.assertListEqual(gc.get_referrers(exc), no_other_refs()) + + async def test_name(self): + name = None + + async def asyncfn(): + nonlocal name + name = asyncio.current_task().get_name() + + async with asyncio.TaskGroup() as tg: + tg.create_task(asyncfn(), name="example name") + + self.assertEqual(name, "example name") async def test_cancels_task_if_created_during_creation(self): @@ -1091,6 +1106,30 @@ async def throw_error(): class TestTaskGroup(BaseTestTaskGroup, unittest.IsolatedAsyncioTestCase): loop_factory = asyncio.EventLoop + @unittest.expectedFailure # TODO: RUSTPYTHON; Test that TaskGroup deletes propagate_cancellation_error + async def test_exception_refcycles_propagate_cancellation_error(self): + return await super().test_exception_refcycles_propagate_cancellation_error() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Test that TaskGroup deletes self._base_error + async def test_exception_refcycles_base_error(self): + return await super().test_exception_refcycles_base_error() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Test that TaskGroup deletes self._errors, and __aexit__ args + async def test_exception_refcycles_errors(self): + return await super().test_exception_refcycles_errors() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Test that TaskGroup deletes self._parent_task + async def test_exception_refcycles_parent_task(self): + return await super().test_exception_refcycles_parent_task() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Test that TaskGroup deletes self._parent_task and create_task() deletes task + async def test_exception_refcycles_parent_task_wr(self): + return await super().test_exception_refcycles_parent_task_wr() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Test that TaskGroup doesn't keep a reference to the raised ExceptionGroup + async def test_exception_refcycles_direct(self): + return await super().test_exception_refcycles_direct() + class TestEagerTaskTaskGroup(BaseTestTaskGroup, unittest.IsolatedAsyncioTestCase): @staticmethod def loop_factory(): @@ -1098,6 +1137,30 @@ def loop_factory(): loop.set_task_factory(asyncio.eager_task_factory) return loop + @unittest.expectedFailure # TODO: RUSTPYTHON; Test that TaskGroup deletes propagate_cancellation_error + async def test_exception_refcycles_propagate_cancellation_error(self): + return await super().test_exception_refcycles_propagate_cancellation_error() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Test that TaskGroup deletes self._base_error + async def test_exception_refcycles_base_error(self): + return await super().test_exception_refcycles_base_error() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Test that TaskGroup deletes self._errors, and __aexit__ args + async def test_exception_refcycles_errors(self): + return await super().test_exception_refcycles_errors() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Test that TaskGroup deletes self._parent_task + async def test_exception_refcycles_parent_task(self): + return await super().test_exception_refcycles_parent_task() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Test that TaskGroup deletes self._parent_task and create_task() deletes task + async def test_exception_refcycles_parent_task_wr(self): + return await super().test_exception_refcycles_parent_task_wr() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Test that TaskGroup doesn't keep a reference to the raised ExceptionGroup + async def test_exception_refcycles_direct(self): + return await super().test_exception_refcycles_direct() + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index 7a830230a7d..8a291f1cb7e 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -20,10 +20,11 @@ from test.test_asyncio import utils as test_utils from test import support from test.support.script_helper import assert_python_ok +from test.support.warnings_helper import ignore_warnings def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) async def coroutine_function(): @@ -86,9 +87,10 @@ class BaseTaskTests: Task = None Future = None + all_tasks = None - def new_task(self, loop, coro, name='TestTask', context=None): - return self.__class__.Task(coro, loop=loop, name=name, context=context) + def new_task(self, loop, coro, name='TestTask', context=None, eager_start=None): + return self.__class__.Task(coro, loop=loop, name=name, context=context, eager_start=eager_start) def new_future(self, loop): return self.__class__.Future(loop=loop) @@ -108,7 +110,7 @@ def test_task_cancel_message_getter(self): async def coro(): pass t = self.new_task(self.loop, coro()) - self.assertTrue(hasattr(t, '_cancel_message')) + self.assertHasAttr(t, '_cancel_message') self.assertEqual(t._cancel_message, None) t.cancel('my message') @@ -544,7 +546,7 @@ async def task(): try: await asyncio.sleep(10) except asyncio.CancelledError: - asyncio.current_task().uncancel() + self.current_task().uncancel() await asyncio.sleep(10) try: @@ -596,7 +598,7 @@ def test_uncancel_structured_blocks(self): loop = asyncio.new_event_loop() async def make_request_with_timeout(*, sleep: float, timeout: float): - task = asyncio.current_task() + task = self.current_task() loop = task.get_loop() timed_out = False @@ -1938,6 +1940,7 @@ async def notmutch(): self.assertFalse(task.cancelled()) self.assertIs(task.exception(), base_exc) + @ignore_warnings(category=DeprecationWarning) def test_iscoroutinefunction(self): def fn(): pass @@ -1955,6 +1958,7 @@ async def fn2(): self.assertFalse(asyncio.iscoroutinefunction(mock.Mock())) self.assertTrue(asyncio.iscoroutinefunction(mock.AsyncMock())) + @ignore_warnings(category=DeprecationWarning) def test_coroutine_non_gen_function(self): async def func(): return 'test' @@ -1983,41 +1987,41 @@ async def coro(): self.assertIsNone(t2.result()) def test_current_task(self): - self.assertIsNone(asyncio.current_task(loop=self.loop)) + self.assertIsNone(self.current_task(loop=self.loop)) async def coro(loop): - self.assertIs(asyncio.current_task(), task) + self.assertIs(self.current_task(), task) - self.assertIs(asyncio.current_task(None), task) - self.assertIs(asyncio.current_task(), task) + self.assertIs(self.current_task(None), task) + self.assertIs(self.current_task(), task) task = self.new_task(self.loop, coro(self.loop)) self.loop.run_until_complete(task) - self.assertIsNone(asyncio.current_task(loop=self.loop)) + self.assertIsNone(self.current_task(loop=self.loop)) def test_current_task_with_interleaving_tasks(self): - self.assertIsNone(asyncio.current_task(loop=self.loop)) + self.assertIsNone(self.current_task(loop=self.loop)) fut1 = self.new_future(self.loop) fut2 = self.new_future(self.loop) async def coro1(loop): - self.assertTrue(asyncio.current_task() is task1) + self.assertTrue(self.current_task() is task1) await fut1 - self.assertTrue(asyncio.current_task() is task1) + self.assertTrue(self.current_task() is task1) fut2.set_result(True) async def coro2(loop): - self.assertTrue(asyncio.current_task() is task2) + self.assertTrue(self.current_task() is task2) fut1.set_result(True) await fut2 - self.assertTrue(asyncio.current_task() is task2) + self.assertTrue(self.current_task() is task2) task1 = self.new_task(self.loop, coro1(self.loop)) task2 = self.new_task(self.loop, coro2(self.loop)) self.loop.run_until_complete(asyncio.wait((task1, task2))) - self.assertIsNone(asyncio.current_task(loop=self.loop)) + self.assertIsNone(self.current_task(loop=self.loop)) # Some thorough tests for cancellation propagation through # coroutines, tasks and wait(). @@ -2112,6 +2116,46 @@ def test_shield_cancel_outer(self): self.assertTrue(outer.cancelled()) self.assertEqual(0, 0 if outer._callbacks is None else len(outer._callbacks)) + def test_shield_cancel_outer_result(self): + mock_handler = mock.Mock() + self.loop.set_exception_handler(mock_handler) + inner = self.new_future(self.loop) + outer = asyncio.shield(inner) + test_utils.run_briefly(self.loop) + outer.cancel() + test_utils.run_briefly(self.loop) + inner.set_result(1) + test_utils.run_briefly(self.loop) + mock_handler.assert_not_called() + + def test_shield_cancel_outer_exception(self): + mock_handler = mock.Mock() + self.loop.set_exception_handler(mock_handler) + inner = self.new_future(self.loop) + outer = asyncio.shield(inner) + test_utils.run_briefly(self.loop) + outer.cancel() + test_utils.run_briefly(self.loop) + inner.set_exception(Exception('foo')) + test_utils.run_briefly(self.loop) + mock_handler.assert_called_once() + + def test_shield_duplicate_log_once(self): + mock_handler = mock.Mock() + self.loop.set_exception_handler(mock_handler) + inner = self.new_future(self.loop) + outer = asyncio.shield(inner) + test_utils.run_briefly(self.loop) + outer.cancel() + test_utils.run_briefly(self.loop) + outer = asyncio.shield(inner) + test_utils.run_briefly(self.loop) + outer.cancel() + test_utils.run_briefly(self.loop) + inner.set_exception(Exception('foo')) + test_utils.run_briefly(self.loop) + mock_handler.assert_called_once() + def test_shield_shortcut(self): fut = self.new_future(self.loop) fut.set_result(42) @@ -2249,9 +2293,7 @@ def test_wait_invalid_args(self): self.assertRaises(ValueError, self.loop.run_until_complete, asyncio.wait([])) - @unittest.skip("RUSTPYTHON: requires cyclic garbage collection") def test_log_destroyed_pending_task(self): - Task = self.__class__.Task async def kill_me(loop): future = self.new_future(loop) @@ -2266,14 +2308,12 @@ async def kill_me(loop): # schedule the task coro = kill_me(self.loop) - task = asyncio.ensure_future(coro, loop=self.loop) - - self.assertEqual(asyncio.all_tasks(loop=self.loop), {task}) + task = self.new_task(self.loop, coro) - asyncio.set_event_loop(None) + self.assertEqual(self.all_tasks(loop=self.loop), {task}) # execute the task so it waits for future - self.loop._run_once() + self.loop.run_until_complete(asyncio.sleep(0)) self.assertEqual(len(self.loop._ready), 0) coro = None @@ -2283,14 +2323,36 @@ async def kill_me(loop): # no more reference to kill_me() task: the task is destroyed by the GC support.gc_collect() - self.assertEqual(asyncio.all_tasks(loop=self.loop), set()) - mock_handler.assert_called_with(self.loop, { 'message': 'Task was destroyed but it is pending!', 'task': mock.ANY, 'source_traceback': source_traceback, }) mock_handler.reset_mock() + # task got resurrected by the exception handler + support.gc_collect() + + self.assertEqual(self.all_tasks(loop=self.loop), set()) + + def test_task_not_crash_without_finalization(self): + Task = self.__class__.Task + + class Subclass(Task): + def __del__(self): + pass + + async def corofn(): + await asyncio.sleep(0.01) + + coro = corofn() + task = Subclass(coro, loop = self.loop) + task._log_destroy_pending = False + + del task + + support.gc_collect() + + coro.close() @mock.patch('asyncio.base_events.logger') def test_tb_logger_not_called_after_cancel(self, m_log): @@ -2432,7 +2494,7 @@ async def coro(): message = m_log.error.call_args[0][0] self.assertIn('Task was destroyed but it is pending', message) - self.assertEqual(asyncio.all_tasks(self.loop), set()) + self.assertEqual(self.all_tasks(self.loop), set()) def test_create_task_with_noncoroutine(self): with self.assertRaisesRegex(TypeError, @@ -2664,6 +2726,35 @@ async def main(): self.assertEqual([None, 1, 2], ret) + def test_eager_start_true(self): + name = None + + async def asyncfn(): + nonlocal name + name = self.current_task().get_name() + + async def main(): + t = self.new_task(coro=asyncfn(), loop=asyncio.get_running_loop(), eager_start=True, name="example") + self.assertTrue(t.done()) + self.assertEqual(name, "example") + await t + + def test_eager_start_false(self): + name = None + + async def asyncfn(): + nonlocal name + name = self.current_task().get_name() + + async def main(): + t = self.new_task(coro=asyncfn(), loop=asyncio.get_running_loop(), eager_start=False, name="example") + self.assertFalse(t.done()) + self.assertIsNone(name) + await t + self.assertEqual(name, "example") + + asyncio.run(main(), loop_factory=asyncio.EventLoop) + def test_get_coro(self): loop = asyncio.new_event_loop() coro = coroutine_function() @@ -2697,12 +2788,12 @@ def __str__(self): coro = coroutine_function() with contextlib.closing(asyncio.EventLoop()) as loop: task = asyncio.Task.__new__(asyncio.Task) - for _ in range(5): with self.assertRaisesRegex(RuntimeError, 'break'): task.__init__(coro, loop=loop, context=obj, name=Break()) coro.close() + task._log_destroy_pending = False del task self.assertEqual(sys.getrefcount(obj), initial_refcount) @@ -2827,6 +2918,8 @@ class CTask_CFuture_Tests(BaseTaskTests, SetMethodsTest, Task = getattr(tasks, '_CTask', None) Future = getattr(futures, '_CFuture', None) + all_tasks = getattr(tasks, '_c_all_tasks', None) + current_task = staticmethod(getattr(tasks, '_c_current_task', None)) @support.refcount_test def test_refleaks_in_task___init__(self): @@ -2849,6 +2942,11 @@ async def coro(): with self.assertRaises(AttributeError): del task._log_destroy_pending + @unittest.expectedFailure # TODO: RUSTPYTHON; Actual: not called. + def test_log_destroyed_pending_task(self): + return super().test_log_destroyed_pending_task() + + @unittest.skipUnless(hasattr(futures, '_CFuture') and hasattr(tasks, '_CTask'), @@ -2858,6 +2956,12 @@ class CTask_CFuture_SubclassTests(BaseTaskTests, test_utils.TestCase): Task = getattr(tasks, '_CTask', None) Future = getattr(futures, '_CFuture', None) + all_tasks = getattr(tasks, '_c_all_tasks', None) + current_task = staticmethod(getattr(tasks, '_c_current_task', None)) + + @unittest.expectedFailure # TODO: RUSTPYTHON; Actual: not called. + def test_log_destroyed_pending_task(self): + return super().test_log_destroyed_pending_task() @unittest.skipUnless(hasattr(tasks, '_CTask'), @@ -2867,6 +2971,12 @@ class CTaskSubclass_PyFuture_Tests(BaseTaskTests, test_utils.TestCase): Task = getattr(tasks, '_CTask', None) Future = futures._PyFuture + all_tasks = getattr(tasks, '_c_all_tasks', None) + current_task = staticmethod(getattr(tasks, '_c_current_task', None)) + + @unittest.expectedFailure # TODO: RUSTPYTHON; Actual: not called. + def test_log_destroyed_pending_task(self): + return super().test_log_destroyed_pending_task() @unittest.skipUnless(hasattr(futures, '_CFuture'), @@ -2876,6 +2986,12 @@ class PyTask_CFutureSubclass_Tests(BaseTaskTests, test_utils.TestCase): Future = getattr(futures, '_CFuture', None) Task = tasks._PyTask + all_tasks = staticmethod(tasks._py_all_tasks) + current_task = staticmethod(tasks._py_current_task) + + @unittest.expectedFailure # TODO: RUSTPYTHON; Actual: not called. + def test_log_destroyed_pending_task(self): + return super().test_log_destroyed_pending_task() @unittest.skipUnless(hasattr(tasks, '_CTask'), @@ -2884,6 +3000,13 @@ class CTask_PyFuture_Tests(BaseTaskTests, test_utils.TestCase): Task = getattr(tasks, '_CTask', None) Future = futures._PyFuture + all_tasks = getattr(tasks, '_c_all_tasks', None) + current_task = staticmethod(getattr(tasks, '_c_current_task', None)) + + @unittest.expectedFailure # TODO: RUSTPYTHON; Actual: not called. + def test_log_destroyed_pending_task(self): + return super().test_log_destroyed_pending_task() + @unittest.skipUnless(hasattr(futures, '_CFuture'), @@ -2892,6 +3015,12 @@ class PyTask_CFuture_Tests(BaseTaskTests, test_utils.TestCase): Task = tasks._PyTask Future = getattr(futures, '_CFuture', None) + all_tasks = staticmethod(tasks._py_all_tasks) + current_task = staticmethod(tasks._py_current_task) + + @unittest.expectedFailure # TODO: RUSTPYTHON; Actual: not called. + def test_log_destroyed_pending_task(self): + return super().test_log_destroyed_pending_task() class PyTask_PyFuture_Tests(BaseTaskTests, SetMethodsTest, @@ -2899,13 +3028,24 @@ class PyTask_PyFuture_Tests(BaseTaskTests, SetMethodsTest, Task = tasks._PyTask Future = futures._PyFuture + all_tasks = staticmethod(tasks._py_all_tasks) + current_task = staticmethod(tasks._py_current_task) + + @unittest.expectedFailure # TODO: RUSTPYTHON; Actual: not called. + def test_log_destroyed_pending_task(self): + return super().test_log_destroyed_pending_task() @add_subclass_tests class PyTask_PyFuture_SubclassTests(BaseTaskTests, test_utils.TestCase): Task = tasks._PyTask Future = futures._PyFuture + all_tasks = staticmethod(tasks._py_all_tasks) + current_task = staticmethod(tasks._py_current_task) + @unittest.expectedFailure # TODO: RUSTPYTHON; Actual: not called. + def test_log_destroyed_pending_task(self): + return super().test_log_destroyed_pending_task() @unittest.skipUnless(hasattr(tasks, '_CTask'), 'requires the C _asyncio module') @@ -2938,6 +3078,7 @@ class BaseTaskIntrospectionTests: _unregister_task = None _enter_task = None _leave_task = None + all_tasks = None def test__register_task_1(self): class TaskLike: @@ -2951,9 +3092,9 @@ def done(self): task = TaskLike() loop = mock.Mock() - self.assertEqual(asyncio.all_tasks(loop), set()) + self.assertEqual(self.all_tasks(loop), set()) self._register_task(task) - self.assertEqual(asyncio.all_tasks(loop), {task}) + self.assertEqual(self.all_tasks(loop), {task}) self._unregister_task(task) def test__register_task_2(self): @@ -2967,9 +3108,9 @@ def done(self): task = TaskLike() loop = mock.Mock() - self.assertEqual(asyncio.all_tasks(loop), set()) + self.assertEqual(self.all_tasks(loop), set()) self._register_task(task) - self.assertEqual(asyncio.all_tasks(loop), {task}) + self.assertEqual(self.all_tasks(loop), {task}) self._unregister_task(task) def test__register_task_3(self): @@ -2983,52 +3124,68 @@ def done(self): task = TaskLike() loop = mock.Mock() - self.assertEqual(asyncio.all_tasks(loop), set()) + self.assertEqual(self.all_tasks(loop), set()) self._register_task(task) - self.assertEqual(asyncio.all_tasks(loop), set()) + self.assertEqual(self.all_tasks(loop), set()) self._unregister_task(task) def test__enter_task(self): task = mock.Mock() loop = mock.Mock() - self.assertIsNone(asyncio.current_task(loop)) + # _enter_task is called by Task.__step while the loop + # is running, so set the loop as the running loop + # for a more realistic test. + asyncio._set_running_loop(loop) + self.assertIsNone(self.current_task(loop)) self._enter_task(loop, task) - self.assertIs(asyncio.current_task(loop), task) + self.assertIs(self.current_task(loop), task) self._leave_task(loop, task) + asyncio._set_running_loop(None) def test__enter_task_failure(self): task1 = mock.Mock() task2 = mock.Mock() loop = mock.Mock() + asyncio._set_running_loop(loop) self._enter_task(loop, task1) with self.assertRaises(RuntimeError): self._enter_task(loop, task2) - self.assertIs(asyncio.current_task(loop), task1) + self.assertIs(self.current_task(loop), task1) self._leave_task(loop, task1) + asyncio._set_running_loop(None) def test__leave_task(self): task = mock.Mock() loop = mock.Mock() + asyncio._set_running_loop(loop) self._enter_task(loop, task) self._leave_task(loop, task) - self.assertIsNone(asyncio.current_task(loop)) + self.assertIsNone(self.current_task(loop)) + asyncio._set_running_loop(None) def test__leave_task_failure1(self): task1 = mock.Mock() task2 = mock.Mock() loop = mock.Mock() + # _leave_task is called by Task.__step while the loop + # is running, so set the loop as the running loop + # for a more realistic test. + asyncio._set_running_loop(loop) self._enter_task(loop, task1) with self.assertRaises(RuntimeError): self._leave_task(loop, task2) - self.assertIs(asyncio.current_task(loop), task1) + self.assertIs(self.current_task(loop), task1) self._leave_task(loop, task1) + asyncio._set_running_loop(None) def test__leave_task_failure2(self): task = mock.Mock() loop = mock.Mock() + asyncio._set_running_loop(loop) with self.assertRaises(RuntimeError): self._leave_task(loop, task) - self.assertIsNone(asyncio.current_task(loop)) + self.assertIsNone(self.current_task(loop)) + asyncio._set_running_loop(None) def test__unregister_task(self): task = mock.Mock() @@ -3036,13 +3193,13 @@ def test__unregister_task(self): task.get_loop = lambda: loop self._register_task(task) self._unregister_task(task) - self.assertEqual(asyncio.all_tasks(loop), set()) + self.assertEqual(self.all_tasks(loop), set()) def test__unregister_task_not_registered(self): task = mock.Mock() loop = mock.Mock() self._unregister_task(task) - self.assertEqual(asyncio.all_tasks(loop), set()) + self.assertEqual(self.all_tasks(loop), set()) class PyIntrospectionTests(test_utils.TestCase, BaseTaskIntrospectionTests): @@ -3050,6 +3207,8 @@ class PyIntrospectionTests(test_utils.TestCase, BaseTaskIntrospectionTests): _unregister_task = staticmethod(tasks._py_unregister_task) _enter_task = staticmethod(tasks._py_enter_task) _leave_task = staticmethod(tasks._py_leave_task) + all_tasks = staticmethod(tasks._py_all_tasks) + current_task = staticmethod(tasks._py_current_task) @unittest.skipUnless(hasattr(tasks, '_c_register_task'), @@ -3060,6 +3219,8 @@ class CIntrospectionTests(test_utils.TestCase, BaseTaskIntrospectionTests): _unregister_task = staticmethod(tasks._c_unregister_task) _enter_task = staticmethod(tasks._c_enter_task) _leave_task = staticmethod(tasks._c_leave_task) + all_tasks = staticmethod(tasks._c_all_tasks) + current_task = staticmethod(tasks._c_current_task) else: _register_task = _unregister_task = _enter_task = _leave_task = None @@ -3117,7 +3278,7 @@ def new_task(self, coro): class GenericTaskTests(test_utils.TestCase): def test_future_subclass(self): - self.assertTrue(issubclass(asyncio.Task, asyncio.Future)) + self.assertIsSubclass(asyncio.Task, asyncio.Future) @support.cpython_only def test_asyncio_module_compiled(self): @@ -3126,14 +3287,14 @@ def test_asyncio_module_compiled(self): # fail on systems where C modules were successfully compiled # (hence the test for _functools etc), but _asyncio somehow didn't. try: - import _functools - import _json - import _pickle + import _functools # noqa: F401 + import _json # noqa: F401 + import _pickle # noqa: F401 except ImportError: self.skipTest('C modules are not available') else: try: - import _asyncio + import _asyncio # noqa: F401 except ImportError: self.fail('_asyncio module is missing') diff --git a/Lib/test/test_asyncio/test_threads.py b/Lib/test/test_asyncio/test_threads.py index 774380270a7..8ad5f9b2c9e 100644 --- a/Lib/test/test_asyncio/test_threads.py +++ b/Lib/test/test_asyncio/test_threads.py @@ -8,7 +8,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class ToThreadTests(unittest.IsolatedAsyncioTestCase): diff --git a/Lib/test/test_asyncio/test_timeouts.py b/Lib/test/test_asyncio/test_timeouts.py index 1f7f9ee696a..f60722c48b7 100644 --- a/Lib/test/test_asyncio/test_timeouts.py +++ b/Lib/test/test_asyncio/test_timeouts.py @@ -9,7 +9,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class TimeoutTests(unittest.IsolatedAsyncioTestCase): @@ -220,7 +220,7 @@ async def test_nested_timeouts_loop_busy(self): # Pretend the loop is busy for a while. time.sleep(0.1) await asyncio.sleep(1) - # TimeoutError was cought by (2) + # TimeoutError was caught by (2) await asyncio.sleep(10) # This sleep should be interrupted by (1) t1 = loop.time() self.assertTrue(t0 <= t1 <= t0 + 1) diff --git a/Lib/test/test_asyncio/test_tools.py b/Lib/test/test_asyncio/test_tools.py new file mode 100644 index 00000000000..34e94830204 --- /dev/null +++ b/Lib/test/test_asyncio/test_tools.py @@ -0,0 +1,1706 @@ +import unittest + +from asyncio import tools + +from collections import namedtuple + +FrameInfo = namedtuple('FrameInfo', ['funcname', 'filename', 'lineno']) +CoroInfo = namedtuple('CoroInfo', ['call_stack', 'task_name']) +TaskInfo = namedtuple('TaskInfo', ['task_id', 'task_name', 'coroutine_stack', 'awaited_by']) +AwaitedInfo = namedtuple('AwaitedInfo', ['thread_id', 'awaited_by']) + + +# mock output of get_all_awaited_by function. +TEST_INPUTS_TREE = [ + [ + # test case containing a task called timer being awaited in two + # different subtasks part of a TaskGroup (root1 and root2) which call + # awaiter functions. + ( + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=3, + task_name="timer", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("awaiter3", "/path/to/app.py", 130), + FrameInfo("awaiter2", "/path/to/app.py", 120), + FrameInfo("awaiter", "/path/to/app.py", 110) + ], + task_name=4 + ), + CoroInfo( + call_stack=[ + FrameInfo("awaiterB3", "/path/to/app.py", 190), + FrameInfo("awaiterB2", "/path/to/app.py", 180), + FrameInfo("awaiterB", "/path/to/app.py", 170) + ], + task_name=5 + ), + CoroInfo( + call_stack=[ + FrameInfo("awaiterB3", "/path/to/app.py", 190), + FrameInfo("awaiterB2", "/path/to/app.py", 180), + FrameInfo("awaiterB", "/path/to/app.py", 170) + ], + task_name=6 + ), + CoroInfo( + call_stack=[ + FrameInfo("awaiter3", "/path/to/app.py", 130), + FrameInfo("awaiter2", "/path/to/app.py", 120), + FrameInfo("awaiter", "/path/to/app.py", 110) + ], + task_name=7 + ) + ] + ), + TaskInfo( + task_id=8, + task_name="root1", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("main", "", 0) + ], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=9, + task_name="root2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("main", "", 0) + ], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=4, + task_name="child1_1", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("blocho_caller", "", 0), + FrameInfo("bloch", "", 0) + ], + task_name=8 + ) + ] + ), + TaskInfo( + task_id=6, + task_name="child2_1", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("blocho_caller", "", 0), + FrameInfo("bloch", "", 0) + ], + task_name=8 + ) + ] + ), + TaskInfo( + task_id=7, + task_name="child1_2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("blocho_caller", "", 0), + FrameInfo("bloch", "", 0) + ], + task_name=9 + ) + ] + ), + TaskInfo( + task_id=5, + task_name="child2_2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("blocho_caller", "", 0), + FrameInfo("bloch", "", 0) + ], + task_name=9 + ) + ] + ) + ] + ), + AwaitedInfo(thread_id=0, awaited_by=[]) + ), + ( + [ + [ + "└── (T) Task-1", + " └── main", + " └── __aexit__", + " └── _aexit", + " ├── (T) root1", + " │ └── bloch", + " │ └── blocho_caller", + " │ └── __aexit__", + " │ └── _aexit", + " │ ├── (T) child1_1", + " │ │ └── awaiter /path/to/app.py:110", + " │ │ └── awaiter2 /path/to/app.py:120", + " │ │ └── awaiter3 /path/to/app.py:130", + " │ │ └── (T) timer", + " │ └── (T) child2_1", + " │ └── awaiterB /path/to/app.py:170", + " │ └── awaiterB2 /path/to/app.py:180", + " │ └── awaiterB3 /path/to/app.py:190", + " │ └── (T) timer", + " └── (T) root2", + " └── bloch", + " └── blocho_caller", + " └── __aexit__", + " └── _aexit", + " ├── (T) child1_2", + " │ └── awaiter /path/to/app.py:110", + " │ └── awaiter2 /path/to/app.py:120", + " │ └── awaiter3 /path/to/app.py:130", + " │ └── (T) timer", + " └── (T) child2_2", + " └── awaiterB /path/to/app.py:170", + " └── awaiterB2 /path/to/app.py:180", + " └── awaiterB3 /path/to/app.py:190", + " └── (T) timer", + ] + ] + ), + ], + [ + # test case containing two roots + ( + AwaitedInfo( + thread_id=9, + awaited_by=[ + TaskInfo( + task_id=5, + task_name="Task-5", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=6, + task_name="Task-6", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main2", "", 0)], + task_name=5 + ) + ] + ), + TaskInfo( + task_id=7, + task_name="Task-7", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main2", "", 0)], + task_name=5 + ) + ] + ), + TaskInfo( + task_id=8, + task_name="Task-8", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main2", "", 0)], + task_name=5 + ) + ] + ) + ] + ), + AwaitedInfo( + thread_id=10, + awaited_by=[ + TaskInfo( + task_id=1, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=2, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=1 + ) + ] + ), + TaskInfo( + task_id=3, + task_name="Task-3", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=1 + ) + ] + ), + TaskInfo( + task_id=4, + task_name="Task-4", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=1 + ) + ] + ) + ] + ), + AwaitedInfo(thread_id=11, awaited_by=[]), + AwaitedInfo(thread_id=0, awaited_by=[]) + ), + ( + [ + [ + "└── (T) Task-5", + " └── main2", + " ├── (T) Task-6", + " ├── (T) Task-7", + " └── (T) Task-8", + ], + [ + "└── (T) Task-1", + " └── main", + " ├── (T) Task-2", + " ├── (T) Task-3", + " └── (T) Task-4", + ], + ] + ), + ], + [ + # test case containing two roots, one of them without subtasks + ( + [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-5", + coroutine_stack=[], + awaited_by=[] + ) + ] + ), + AwaitedInfo( + thread_id=3, + awaited_by=[ + TaskInfo( + task_id=4, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=5, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=4 + ) + ] + ), + TaskInfo( + task_id=6, + task_name="Task-3", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=4 + ) + ] + ), + TaskInfo( + task_id=7, + task_name="Task-4", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=4 + ) + ] + ) + ] + ), + AwaitedInfo(thread_id=8, awaited_by=[]), + AwaitedInfo(thread_id=0, awaited_by=[]) + ] + ), + ( + [ + ["└── (T) Task-5"], + [ + "└── (T) Task-1", + " └── main", + " ├── (T) Task-2", + " ├── (T) Task-3", + " └── (T) Task-4", + ], + ] + ), + ], +] + +TEST_INPUTS_CYCLES_TREE = [ + [ + # this test case contains a cycle: two tasks awaiting each other. + ( + [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=3, + task_name="a", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("awaiter2", "", 0)], + task_name=4 + ), + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=4, + task_name="b", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("awaiter", "", 0)], + task_name=3 + ) + ] + ) + ] + ), + AwaitedInfo(thread_id=0, awaited_by=[]) + ] + ), + ([[4, 3, 4]]), + ], + [ + # this test case contains two cycles + ( + [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=3, + task_name="A", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0), + FrameInfo("task_b", "", 0) + ], + task_name=4 + ) + ] + ), + TaskInfo( + task_id=4, + task_name="B", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0), + FrameInfo("task_c", "", 0) + ], + task_name=5 + ), + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0), + FrameInfo("task_a", "", 0) + ], + task_name=3 + ) + ] + ), + TaskInfo( + task_id=5, + task_name="C", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0) + ], + task_name=6 + ) + ] + ), + TaskInfo( + task_id=6, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0), + FrameInfo("task_b", "", 0) + ], + task_name=4 + ) + ] + ) + ] + ), + AwaitedInfo(thread_id=0, awaited_by=[]) + ] + ), + ([[4, 3, 4], [4, 6, 5, 4]]), + ], +] + +TEST_INPUTS_TABLE = [ + [ + # test case containing a task called timer being awaited in two + # different subtasks part of a TaskGroup (root1 and root2) which call + # awaiter functions. + ( + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=3, + task_name="timer", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("awaiter3", "", 0), + FrameInfo("awaiter2", "", 0), + FrameInfo("awaiter", "", 0) + ], + task_name=4 + ), + CoroInfo( + call_stack=[ + FrameInfo("awaiter1_3", "", 0), + FrameInfo("awaiter1_2", "", 0), + FrameInfo("awaiter1", "", 0) + ], + task_name=5 + ), + CoroInfo( + call_stack=[ + FrameInfo("awaiter1_3", "", 0), + FrameInfo("awaiter1_2", "", 0), + FrameInfo("awaiter1", "", 0) + ], + task_name=6 + ), + CoroInfo( + call_stack=[ + FrameInfo("awaiter3", "", 0), + FrameInfo("awaiter2", "", 0), + FrameInfo("awaiter", "", 0) + ], + task_name=7 + ) + ] + ), + TaskInfo( + task_id=8, + task_name="root1", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("main", "", 0) + ], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=9, + task_name="root2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("main", "", 0) + ], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=4, + task_name="child1_1", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("blocho_caller", "", 0), + FrameInfo("bloch", "", 0) + ], + task_name=8 + ) + ] + ), + TaskInfo( + task_id=6, + task_name="child2_1", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("blocho_caller", "", 0), + FrameInfo("bloch", "", 0) + ], + task_name=8 + ) + ] + ), + TaskInfo( + task_id=7, + task_name="child1_2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("blocho_caller", "", 0), + FrameInfo("bloch", "", 0) + ], + task_name=9 + ) + ] + ), + TaskInfo( + task_id=5, + task_name="child2_2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("blocho_caller", "", 0), + FrameInfo("bloch", "", 0) + ], + task_name=9 + ) + ] + ) + ] + ), + AwaitedInfo(thread_id=0, awaited_by=[]) + ), + ( + [ + [1, "0x2", "Task-1", "", "", "", "0x0"], + [ + 1, + "0x3", + "timer", + "", + "awaiter3 -> awaiter2 -> awaiter", + "child1_1", + "0x4", + ], + [ + 1, + "0x3", + "timer", + "", + "awaiter1_3 -> awaiter1_2 -> awaiter1", + "child2_2", + "0x5", + ], + [ + 1, + "0x3", + "timer", + "", + "awaiter1_3 -> awaiter1_2 -> awaiter1", + "child2_1", + "0x6", + ], + [ + 1, + "0x3", + "timer", + "", + "awaiter3 -> awaiter2 -> awaiter", + "child1_2", + "0x7", + ], + [ + 1, + "0x8", + "root1", + "", + "_aexit -> __aexit__ -> main", + "Task-1", + "0x2", + ], + [ + 1, + "0x9", + "root2", + "", + "_aexit -> __aexit__ -> main", + "Task-1", + "0x2", + ], + [ + 1, + "0x4", + "child1_1", + "", + "_aexit -> __aexit__ -> blocho_caller -> bloch", + "root1", + "0x8", + ], + [ + 1, + "0x6", + "child2_1", + "", + "_aexit -> __aexit__ -> blocho_caller -> bloch", + "root1", + "0x8", + ], + [ + 1, + "0x7", + "child1_2", + "", + "_aexit -> __aexit__ -> blocho_caller -> bloch", + "root2", + "0x9", + ], + [ + 1, + "0x5", + "child2_2", + "", + "_aexit -> __aexit__ -> blocho_caller -> bloch", + "root2", + "0x9", + ], + ] + ), + ], + [ + # test case containing two roots + ( + AwaitedInfo( + thread_id=9, + awaited_by=[ + TaskInfo( + task_id=5, + task_name="Task-5", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=6, + task_name="Task-6", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main2", "", 0)], + task_name=5 + ) + ] + ), + TaskInfo( + task_id=7, + task_name="Task-7", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main2", "", 0)], + task_name=5 + ) + ] + ), + TaskInfo( + task_id=8, + task_name="Task-8", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main2", "", 0)], + task_name=5 + ) + ] + ) + ] + ), + AwaitedInfo( + thread_id=10, + awaited_by=[ + TaskInfo( + task_id=1, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=2, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=1 + ) + ] + ), + TaskInfo( + task_id=3, + task_name="Task-3", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=1 + ) + ] + ), + TaskInfo( + task_id=4, + task_name="Task-4", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=1 + ) + ] + ) + ] + ), + AwaitedInfo(thread_id=11, awaited_by=[]), + AwaitedInfo(thread_id=0, awaited_by=[]) + ), + ( + [ + [9, "0x5", "Task-5", "", "", "", "0x0"], + [9, "0x6", "Task-6", "", "main2", "Task-5", "0x5"], + [9, "0x7", "Task-7", "", "main2", "Task-5", "0x5"], + [9, "0x8", "Task-8", "", "main2", "Task-5", "0x5"], + [10, "0x1", "Task-1", "", "", "", "0x0"], + [10, "0x2", "Task-2", "", "main", "Task-1", "0x1"], + [10, "0x3", "Task-3", "", "main", "Task-1", "0x1"], + [10, "0x4", "Task-4", "", "main", "Task-1", "0x1"], + ] + ), + ], + [ + # test case containing two roots, one of them without subtasks + ( + [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-5", + coroutine_stack=[], + awaited_by=[] + ) + ] + ), + AwaitedInfo( + thread_id=3, + awaited_by=[ + TaskInfo( + task_id=4, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=5, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=4 + ) + ] + ), + TaskInfo( + task_id=6, + task_name="Task-3", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=4 + ) + ] + ), + TaskInfo( + task_id=7, + task_name="Task-4", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=4 + ) + ] + ) + ] + ), + AwaitedInfo(thread_id=8, awaited_by=[]), + AwaitedInfo(thread_id=0, awaited_by=[]) + ] + ), + ( + [ + [1, "0x2", "Task-5", "", "", "", "0x0"], + [3, "0x4", "Task-1", "", "", "", "0x0"], + [3, "0x5", "Task-2", "", "main", "Task-1", "0x4"], + [3, "0x6", "Task-3", "", "main", "Task-1", "0x4"], + [3, "0x7", "Task-4", "", "main", "Task-1", "0x4"], + ] + ), + ], + # CASES WITH CYCLES + [ + # this test case contains a cycle: two tasks awaiting each other. + ( + [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=3, + task_name="a", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("awaiter2", "", 0)], + task_name=4 + ), + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=4, + task_name="b", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("awaiter", "", 0)], + task_name=3 + ) + ] + ) + ] + ), + AwaitedInfo(thread_id=0, awaited_by=[]) + ] + ), + ( + [ + [1, "0x2", "Task-1", "", "", "", "0x0"], + [1, "0x3", "a", "", "awaiter2", "b", "0x4"], + [1, "0x3", "a", "", "main", "Task-1", "0x2"], + [1, "0x4", "b", "", "awaiter", "a", "0x3"], + ] + ), + ], + [ + # this test case contains two cycles + ( + [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=3, + task_name="A", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0), + FrameInfo("task_b", "", 0) + ], + task_name=4 + ) + ] + ), + TaskInfo( + task_id=4, + task_name="B", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0), + FrameInfo("task_c", "", 0) + ], + task_name=5 + ), + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0), + FrameInfo("task_a", "", 0) + ], + task_name=3 + ) + ] + ), + TaskInfo( + task_id=5, + task_name="C", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0) + ], + task_name=6 + ) + ] + ), + TaskInfo( + task_id=6, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0), + FrameInfo("task_b", "", 0) + ], + task_name=4 + ) + ] + ) + ] + ), + AwaitedInfo(thread_id=0, awaited_by=[]) + ] + ), + ( + [ + [1, "0x2", "Task-1", "", "", "", "0x0"], + [ + 1, + "0x3", + "A", + "", + "nested -> nested -> task_b", + "B", + "0x4", + ], + [ + 1, + "0x4", + "B", + "", + "nested -> nested -> task_c", + "C", + "0x5", + ], + [ + 1, + "0x4", + "B", + "", + "nested -> nested -> task_a", + "A", + "0x3", + ], + [ + 1, + "0x5", + "C", + "", + "nested -> nested", + "Task-2", + "0x6", + ], + [ + 1, + "0x6", + "Task-2", + "", + "nested -> nested -> task_b", + "B", + "0x4", + ], + ] + ), + ], +] + + +class TestAsyncioToolsTree(unittest.TestCase): + def test_asyncio_utils(self): + for input_, tree in TEST_INPUTS_TREE: + with self.subTest(input_): + result = tools.build_async_tree(input_) + self.assertEqual(result, tree) + + def test_asyncio_utils_cycles(self): + for input_, cycles in TEST_INPUTS_CYCLES_TREE: + with self.subTest(input_): + try: + tools.build_async_tree(input_) + except tools.CycleFoundException as e: + self.assertEqual(e.cycles, cycles) + + +class TestAsyncioToolsTable(unittest.TestCase): + def test_asyncio_utils(self): + for input_, table in TEST_INPUTS_TABLE: + with self.subTest(input_): + result = tools.build_task_table(input_) + self.assertEqual(result, table) + + +class TestAsyncioToolsBasic(unittest.TestCase): + def test_empty_input_tree(self): + """Test build_async_tree with empty input.""" + result = [] + expected_output = [] + self.assertEqual(tools.build_async_tree(result), expected_output) + + def test_empty_input_table(self): + """Test build_task_table with empty input.""" + result = [] + expected_output = [] + self.assertEqual(tools.build_task_table(result), expected_output) + + def test_only_independent_tasks_tree(self): + input_ = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=10, + task_name="taskA", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=11, + task_name="taskB", + coroutine_stack=[], + awaited_by=[] + ) + ] + ) + ] + expected = [["└── (T) taskA"], ["└── (T) taskB"]] + result = tools.build_async_tree(input_) + self.assertEqual(sorted(result), sorted(expected)) + + def test_only_independent_tasks_table(self): + input_ = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=10, + task_name="taskA", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=11, + task_name="taskB", + coroutine_stack=[], + awaited_by=[] + ) + ] + ) + ] + self.assertEqual( + tools.build_task_table(input_), + [[1, '0xa', 'taskA', '', '', '', '0x0'], [1, '0xb', 'taskB', '', '', '', '0x0']] + ) + + def test_single_task_tree(self): + """Test build_async_tree with a single task and no awaits.""" + result = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ) + ] + ) + ] + expected_output = [ + [ + "└── (T) Task-1", + ] + ] + self.assertEqual(tools.build_async_tree(result), expected_output) + + def test_single_task_table(self): + """Test build_task_table with a single task and no awaits.""" + result = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ) + ] + ) + ] + expected_output = [[1, '0x2', 'Task-1', '', '', '', '0x0']] + self.assertEqual(tools.build_task_table(result), expected_output) + + def test_cycle_detection(self): + """Test build_async_tree raises CycleFoundException for cyclic input.""" + result = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=3 + ) + ] + ), + TaskInfo( + task_id=3, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=2 + ) + ] + ) + ] + ) + ] + with self.assertRaises(tools.CycleFoundException) as context: + tools.build_async_tree(result) + self.assertEqual(context.exception.cycles, [[3, 2, 3]]) + + def test_complex_tree(self): + """Test build_async_tree with a more complex tree structure.""" + result = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=3, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=4, + task_name="Task-3", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=3 + ) + ] + ) + ] + ) + ] + expected_output = [ + [ + "└── (T) Task-1", + " └── main", + " └── (T) Task-2", + " └── main", + " └── (T) Task-3", + ] + ] + self.assertEqual(tools.build_async_tree(result), expected_output) + + def test_complex_table(self): + """Test build_task_table with a more complex tree structure.""" + result = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=3, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=4, + task_name="Task-3", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=3 + ) + ] + ) + ] + ) + ] + expected_output = [ + [1, '0x2', 'Task-1', '', '', '', '0x0'], + [1, '0x3', 'Task-2', '', 'main', 'Task-1', '0x2'], + [1, '0x4', 'Task-3', '', 'main', 'Task-2', '0x3'] + ] + self.assertEqual(tools.build_task_table(result), expected_output) + + def test_deep_coroutine_chain(self): + input_ = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=10, + task_name="leaf", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("c1", "", 0), + FrameInfo("c2", "", 0), + FrameInfo("c3", "", 0), + FrameInfo("c4", "", 0), + FrameInfo("c5", "", 0) + ], + task_name=11 + ) + ] + ), + TaskInfo( + task_id=11, + task_name="root", + coroutine_stack=[], + awaited_by=[] + ) + ] + ) + ] + expected = [ + [ + "└── (T) root", + " └── c5", + " └── c4", + " └── c3", + " └── c2", + " └── c1", + " └── (T) leaf", + ] + ] + result = tools.build_async_tree(input_) + self.assertEqual(result, expected) + + def test_multiple_cycles_same_node(self): + input_ = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=1, + task_name="Task-A", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("call1", "", 0)], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=2, + task_name="Task-B", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("call2", "", 0)], + task_name=3 + ) + ] + ), + TaskInfo( + task_id=3, + task_name="Task-C", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("call3", "", 0)], + task_name=1 + ), + CoroInfo( + call_stack=[FrameInfo("call4", "", 0)], + task_name=2 + ) + ] + ) + ] + ) + ] + with self.assertRaises(tools.CycleFoundException) as ctx: + tools.build_async_tree(input_) + cycles = ctx.exception.cycles + self.assertTrue(any(set(c) == {1, 2, 3} for c in cycles)) + + def test_table_output_format(self): + input_ = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=1, + task_name="Task-A", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("foo", "", 0)], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=2, + task_name="Task-B", + coroutine_stack=[], + awaited_by=[] + ) + ] + ) + ] + table = tools.build_task_table(input_) + for row in table: + self.assertEqual(len(row), 7) + self.assertIsInstance(row[0], int) # thread ID + self.assertTrue( + isinstance(row[1], str) and row[1].startswith("0x") + ) # hex task ID + self.assertIsInstance(row[2], str) # task name + self.assertIsInstance(row[3], str) # coroutine stack + self.assertIsInstance(row[4], str) # coroutine chain + self.assertIsInstance(row[5], str) # awaiter name + self.assertTrue( + isinstance(row[6], str) and row[6].startswith("0x") + ) # hex awaiter ID + + +class TestAsyncioToolsEdgeCases(unittest.TestCase): + + def test_task_awaits_self(self): + """A task directly awaits itself - should raise a cycle.""" + input_ = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=1, + task_name="Self-Awaiter", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("loopback", "", 0)], + task_name=1 + ) + ] + ) + ] + ) + ] + with self.assertRaises(tools.CycleFoundException) as ctx: + tools.build_async_tree(input_) + self.assertIn([1, 1], ctx.exception.cycles) + + def test_task_with_missing_awaiter_id(self): + """Awaiter ID not in task list - should not crash, just show 'Unknown'.""" + input_ = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=1, + task_name="Task-A", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("coro", "", 0)], + task_name=999 + ) + ] + ) + ] + ) + ] + table = tools.build_task_table(input_) + self.assertEqual(len(table), 1) + self.assertEqual(table[0][5], "Unknown") + + def test_duplicate_coroutine_frames(self): + """Same coroutine frame repeated under a parent - should deduplicate.""" + input_ = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=1, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("frameA", "", 0)], + task_name=2 + ), + CoroInfo( + call_stack=[FrameInfo("frameA", "", 0)], + task_name=3 + ) + ] + ), + TaskInfo( + task_id=2, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=3, + task_name="Task-3", + coroutine_stack=[], + awaited_by=[] + ) + ] + ) + ] + tree = tools.build_async_tree(input_) + # Both children should be under the same coroutine node + flat = "\n".join(tree[0]) + self.assertIn("frameA", flat) + self.assertIn("Task-2", flat) + self.assertIn("Task-1", flat) + + flat = "\n".join(tree[1]) + self.assertIn("frameA", flat) + self.assertIn("Task-3", flat) + self.assertIn("Task-1", flat) + + def test_task_with_no_name(self): + """Task with no name in id2name - should still render with fallback.""" + input_ = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=1, + task_name="root", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("f1", "", 0)], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=2, + task_name=None, + coroutine_stack=[], + awaited_by=[] + ) + ] + ) + ] + # If name is None, fallback to string should not crash + tree = tools.build_async_tree(input_) + self.assertIn("(T) None", "\n".join(tree[0])) + + def test_tree_rendering_with_custom_emojis(self): + """Pass custom emojis to the tree renderer.""" + input_ = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=1, + task_name="MainTask", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("f1", "", 0), + FrameInfo("f2", "", 0) + ], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=2, + task_name="SubTask", + coroutine_stack=[], + awaited_by=[] + ) + ] + ) + ] + tree = tools.build_async_tree(input_, task_emoji="🧵", cor_emoji="🔁") + flat = "\n".join(tree[0]) + self.assertIn("🧵 MainTask", flat) + self.assertIn("🔁 f1", flat) + self.assertIn("🔁 f2", flat) + self.assertIn("🧵 SubTask", flat) diff --git a/Lib/test/test_asyncio/test_transports.py b/Lib/test/test_asyncio/test_transports.py index bbdb218efaa..dbb572e2e15 100644 --- a/Lib/test/test_asyncio/test_transports.py +++ b/Lib/test/test_asyncio/test_transports.py @@ -10,7 +10,7 @@ def tearDownModule(): # not needed for the test file but added for uniformness with all other # asyncio test files for the sake of unified cleanup - asyncio.set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class TransportTests(unittest.TestCase): diff --git a/Lib/test/test_asyncio/test_unix_events.py b/Lib/test/test_asyncio/test_unix_events.py index 0e5488da272..520f5c733c3 100644 --- a/Lib/test/test_asyncio/test_unix_events.py +++ b/Lib/test/test_asyncio/test_unix_events.py @@ -10,11 +10,9 @@ import socket import stat import sys -import threading import time import unittest from unittest import mock -import warnings from test import support from test.support import os_helper @@ -27,13 +25,12 @@ import asyncio -from asyncio import log from asyncio import unix_events from test.test_asyncio import utils as test_utils def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) MOCK_ANY = mock.ANY @@ -1112,697 +1109,6 @@ def test_write_eof_pending(self): self.assertFalse(self.protocol.connection_lost.called) -class AbstractChildWatcherTests(unittest.TestCase): - - def test_warns_on_subclassing(self): - with self.assertWarns(DeprecationWarning): - class MyWatcher(asyncio.AbstractChildWatcher): - pass - - def test_not_implemented(self): - f = mock.Mock() - watcher = asyncio.AbstractChildWatcher() - self.assertRaises( - NotImplementedError, watcher.add_child_handler, f, f) - self.assertRaises( - NotImplementedError, watcher.remove_child_handler, f) - self.assertRaises( - NotImplementedError, watcher.attach_loop, f) - self.assertRaises( - NotImplementedError, watcher.close) - self.assertRaises( - NotImplementedError, watcher.is_active) - self.assertRaises( - NotImplementedError, watcher.__enter__) - self.assertRaises( - NotImplementedError, watcher.__exit__, f, f, f) - - -class BaseChildWatcherTests(unittest.TestCase): - - def test_not_implemented(self): - f = mock.Mock() - watcher = unix_events.BaseChildWatcher() - self.assertRaises( - NotImplementedError, watcher._do_waitpid, f) - - -class ChildWatcherTestsMixin: - - ignore_warnings = mock.patch.object(log.logger, "warning") - - def setUp(self): - super().setUp() - self.loop = self.new_test_loop() - self.running = False - self.zombies = {} - - with mock.patch.object( - self.loop, "add_signal_handler") as self.m_add_signal_handler: - self.watcher = self.create_watcher() - self.watcher.attach_loop(self.loop) - - def waitpid(self, pid, flags): - if isinstance(self.watcher, asyncio.SafeChildWatcher) or pid != -1: - self.assertGreater(pid, 0) - try: - if pid < 0: - return self.zombies.popitem() - else: - return pid, self.zombies.pop(pid) - except KeyError: - pass - if self.running: - return 0, 0 - else: - raise ChildProcessError() - - def add_zombie(self, pid, status): - self.zombies[pid] = status - - def waitstatus_to_exitcode(self, status): - if status > 32768: - return status - 32768 - elif 32700 < status < 32768: - return status - 32768 - else: - return status - - def test_create_watcher(self): - self.m_add_signal_handler.assert_called_once_with( - signal.SIGCHLD, self.watcher._sig_chld) - - def waitpid_mocks(func): - def wrapped_func(self): - def patch(target, wrapper): - return mock.patch(target, wraps=wrapper, - new_callable=mock.Mock) - - with patch('asyncio.unix_events.waitstatus_to_exitcode', self.waitstatus_to_exitcode), \ - patch('os.waitpid', self.waitpid) as m_waitpid: - func(self, m_waitpid) - return wrapped_func - - @waitpid_mocks - def test_sigchld(self, m_waitpid): - # register a child - callback = mock.Mock() - - with self.watcher: - self.running = True - self.watcher.add_child_handler(42, callback, 9, 10, 14) - - self.assertFalse(callback.called) - - # child is running - self.watcher._sig_chld() - - self.assertFalse(callback.called) - - # child terminates (returncode 12) - self.running = False - self.add_zombie(42, EXITCODE(12)) - self.watcher._sig_chld() - - callback.assert_called_once_with(42, 12, 9, 10, 14) - - callback.reset_mock() - - # ensure that the child is effectively reaped - self.add_zombie(42, EXITCODE(13)) - with self.ignore_warnings: - self.watcher._sig_chld() - - self.assertFalse(callback.called) - - # sigchld called again - self.zombies.clear() - self.watcher._sig_chld() - - self.assertFalse(callback.called) - - @waitpid_mocks - def test_sigchld_two_children(self, m_waitpid): - callback1 = mock.Mock() - callback2 = mock.Mock() - - # register child 1 - with self.watcher: - self.running = True - self.watcher.add_child_handler(43, callback1, 7, 8) - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - - # register child 2 - with self.watcher: - self.watcher.add_child_handler(44, callback2, 147, 18) - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - - # children are running - self.watcher._sig_chld() - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - - # child 1 terminates (signal 3) - self.add_zombie(43, SIGNAL(3)) - self.watcher._sig_chld() - - callback1.assert_called_once_with(43, -3, 7, 8) - self.assertFalse(callback2.called) - - callback1.reset_mock() - - # child 2 still running - self.watcher._sig_chld() - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - - # child 2 terminates (code 108) - self.add_zombie(44, EXITCODE(108)) - self.running = False - self.watcher._sig_chld() - - callback2.assert_called_once_with(44, 108, 147, 18) - self.assertFalse(callback1.called) - - callback2.reset_mock() - - # ensure that the children are effectively reaped - self.add_zombie(43, EXITCODE(14)) - self.add_zombie(44, EXITCODE(15)) - with self.ignore_warnings: - self.watcher._sig_chld() - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - - # sigchld called again - self.zombies.clear() - self.watcher._sig_chld() - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - - @waitpid_mocks - def test_sigchld_two_children_terminating_together(self, m_waitpid): - callback1 = mock.Mock() - callback2 = mock.Mock() - - # register child 1 - with self.watcher: - self.running = True - self.watcher.add_child_handler(45, callback1, 17, 8) - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - - # register child 2 - with self.watcher: - self.watcher.add_child_handler(46, callback2, 1147, 18) - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - - # children are running - self.watcher._sig_chld() - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - - # child 1 terminates (code 78) - # child 2 terminates (signal 5) - self.add_zombie(45, EXITCODE(78)) - self.add_zombie(46, SIGNAL(5)) - self.running = False - self.watcher._sig_chld() - - callback1.assert_called_once_with(45, 78, 17, 8) - callback2.assert_called_once_with(46, -5, 1147, 18) - - callback1.reset_mock() - callback2.reset_mock() - - # ensure that the children are effectively reaped - self.add_zombie(45, EXITCODE(14)) - self.add_zombie(46, EXITCODE(15)) - with self.ignore_warnings: - self.watcher._sig_chld() - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - - @waitpid_mocks - def test_sigchld_race_condition(self, m_waitpid): - # register a child - callback = mock.Mock() - - with self.watcher: - # child terminates before being registered - self.add_zombie(50, EXITCODE(4)) - self.watcher._sig_chld() - - self.watcher.add_child_handler(50, callback, 1, 12) - - callback.assert_called_once_with(50, 4, 1, 12) - callback.reset_mock() - - # ensure that the child is effectively reaped - self.add_zombie(50, SIGNAL(1)) - with self.ignore_warnings: - self.watcher._sig_chld() - - self.assertFalse(callback.called) - - @waitpid_mocks - def test_sigchld_replace_handler(self, m_waitpid): - callback1 = mock.Mock() - callback2 = mock.Mock() - - # register a child - with self.watcher: - self.running = True - self.watcher.add_child_handler(51, callback1, 19) - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - - # register the same child again - with self.watcher: - self.watcher.add_child_handler(51, callback2, 21) - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - - # child terminates (signal 8) - self.running = False - self.add_zombie(51, SIGNAL(8)) - self.watcher._sig_chld() - - callback2.assert_called_once_with(51, -8, 21) - self.assertFalse(callback1.called) - - callback2.reset_mock() - - # ensure that the child is effectively reaped - self.add_zombie(51, EXITCODE(13)) - with self.ignore_warnings: - self.watcher._sig_chld() - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - - @waitpid_mocks - def test_sigchld_remove_handler(self, m_waitpid): - callback = mock.Mock() - - # register a child - with self.watcher: - self.running = True - self.watcher.add_child_handler(52, callback, 1984) - - self.assertFalse(callback.called) - - # unregister the child - self.watcher.remove_child_handler(52) - - self.assertFalse(callback.called) - - # child terminates (code 99) - self.running = False - self.add_zombie(52, EXITCODE(99)) - with self.ignore_warnings: - self.watcher._sig_chld() - - self.assertFalse(callback.called) - - @waitpid_mocks - def test_sigchld_unknown_status(self, m_waitpid): - callback = mock.Mock() - - # register a child - with self.watcher: - self.running = True - self.watcher.add_child_handler(53, callback, -19) - - self.assertFalse(callback.called) - - # terminate with unknown status - self.zombies[53] = 1178 - self.running = False - self.watcher._sig_chld() - - callback.assert_called_once_with(53, 1178, -19) - - callback.reset_mock() - - # ensure that the child is effectively reaped - self.add_zombie(53, EXITCODE(101)) - with self.ignore_warnings: - self.watcher._sig_chld() - - self.assertFalse(callback.called) - - @waitpid_mocks - def test_remove_child_handler(self, m_waitpid): - callback1 = mock.Mock() - callback2 = mock.Mock() - callback3 = mock.Mock() - - # register children - with self.watcher: - self.running = True - self.watcher.add_child_handler(54, callback1, 1) - self.watcher.add_child_handler(55, callback2, 2) - self.watcher.add_child_handler(56, callback3, 3) - - # remove child handler 1 - self.assertTrue(self.watcher.remove_child_handler(54)) - - # remove child handler 2 multiple times - self.assertTrue(self.watcher.remove_child_handler(55)) - self.assertFalse(self.watcher.remove_child_handler(55)) - self.assertFalse(self.watcher.remove_child_handler(55)) - - # all children terminate - self.add_zombie(54, EXITCODE(0)) - self.add_zombie(55, EXITCODE(1)) - self.add_zombie(56, EXITCODE(2)) - self.running = False - with self.ignore_warnings: - self.watcher._sig_chld() - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - callback3.assert_called_once_with(56, 2, 3) - - @waitpid_mocks - def test_sigchld_unhandled_exception(self, m_waitpid): - callback = mock.Mock() - - # register a child - with self.watcher: - self.running = True - self.watcher.add_child_handler(57, callback) - - # raise an exception - m_waitpid.side_effect = ValueError - - with mock.patch.object(log.logger, - 'error') as m_error: - - self.assertEqual(self.watcher._sig_chld(), None) - self.assertTrue(m_error.called) - - @waitpid_mocks - def test_sigchld_child_reaped_elsewhere(self, m_waitpid): - # register a child - callback = mock.Mock() - - with self.watcher: - self.running = True - self.watcher.add_child_handler(58, callback) - - self.assertFalse(callback.called) - - # child terminates - self.running = False - self.add_zombie(58, EXITCODE(4)) - - # waitpid is called elsewhere - os.waitpid(58, os.WNOHANG) - - m_waitpid.reset_mock() - - # sigchld - with self.ignore_warnings: - self.watcher._sig_chld() - - if isinstance(self.watcher, asyncio.FastChildWatcher): - # here the FastChildWatcher enters a deadlock - # (there is no way to prevent it) - self.assertFalse(callback.called) - else: - callback.assert_called_once_with(58, 255) - - @waitpid_mocks - def test_sigchld_unknown_pid_during_registration(self, m_waitpid): - # register two children - callback1 = mock.Mock() - callback2 = mock.Mock() - - with self.ignore_warnings, self.watcher: - self.running = True - # child 1 terminates - self.add_zombie(591, EXITCODE(7)) - # an unknown child terminates - self.add_zombie(593, EXITCODE(17)) - - self.watcher._sig_chld() - - self.watcher.add_child_handler(591, callback1) - self.watcher.add_child_handler(592, callback2) - - callback1.assert_called_once_with(591, 7) - self.assertFalse(callback2.called) - - @waitpid_mocks - def test_set_loop(self, m_waitpid): - # register a child - callback = mock.Mock() - - with self.watcher: - self.running = True - self.watcher.add_child_handler(60, callback) - - # attach a new loop - old_loop = self.loop - self.loop = self.new_test_loop() - patch = mock.patch.object - - with patch(old_loop, "remove_signal_handler") as m_old_remove, \ - patch(self.loop, "add_signal_handler") as m_new_add: - - self.watcher.attach_loop(self.loop) - - m_old_remove.assert_called_once_with( - signal.SIGCHLD) - m_new_add.assert_called_once_with( - signal.SIGCHLD, self.watcher._sig_chld) - - # child terminates - self.running = False - self.add_zombie(60, EXITCODE(9)) - self.watcher._sig_chld() - - callback.assert_called_once_with(60, 9) - - @waitpid_mocks - def test_set_loop_race_condition(self, m_waitpid): - # register 3 children - callback1 = mock.Mock() - callback2 = mock.Mock() - callback3 = mock.Mock() - - with self.watcher: - self.running = True - self.watcher.add_child_handler(61, callback1) - self.watcher.add_child_handler(62, callback2) - self.watcher.add_child_handler(622, callback3) - - # detach the loop - old_loop = self.loop - self.loop = None - - with mock.patch.object( - old_loop, "remove_signal_handler") as m_remove_signal_handler: - - with self.assertWarnsRegex( - RuntimeWarning, 'A loop is being detached'): - self.watcher.attach_loop(None) - - m_remove_signal_handler.assert_called_once_with( - signal.SIGCHLD) - - # child 1 & 2 terminate - self.add_zombie(61, EXITCODE(11)) - self.add_zombie(62, SIGNAL(5)) - - # SIGCHLD was not caught - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - self.assertFalse(callback3.called) - - # attach a new loop - self.loop = self.new_test_loop() - - with mock.patch.object( - self.loop, "add_signal_handler") as m_add_signal_handler: - - self.watcher.attach_loop(self.loop) - - m_add_signal_handler.assert_called_once_with( - signal.SIGCHLD, self.watcher._sig_chld) - callback1.assert_called_once_with(61, 11) # race condition! - callback2.assert_called_once_with(62, -5) # race condition! - self.assertFalse(callback3.called) - - callback1.reset_mock() - callback2.reset_mock() - - # child 3 terminates - self.running = False - self.add_zombie(622, EXITCODE(19)) - self.watcher._sig_chld() - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - callback3.assert_called_once_with(622, 19) - - @waitpid_mocks - def test_close(self, m_waitpid): - # register two children - callback1 = mock.Mock() - - with self.watcher: - self.running = True - # child 1 terminates - self.add_zombie(63, EXITCODE(9)) - # other child terminates - self.add_zombie(65, EXITCODE(18)) - self.watcher._sig_chld() - - self.watcher.add_child_handler(63, callback1) - self.watcher.add_child_handler(64, callback1) - - self.assertEqual(len(self.watcher._callbacks), 1) - if isinstance(self.watcher, asyncio.FastChildWatcher): - self.assertEqual(len(self.watcher._zombies), 1) - - with mock.patch.object( - self.loop, - "remove_signal_handler") as m_remove_signal_handler: - - self.watcher.close() - - m_remove_signal_handler.assert_called_once_with( - signal.SIGCHLD) - self.assertFalse(self.watcher._callbacks) - if isinstance(self.watcher, asyncio.FastChildWatcher): - self.assertFalse(self.watcher._zombies) - - -class SafeChildWatcherTests (ChildWatcherTestsMixin, test_utils.TestCase): - def create_watcher(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - return asyncio.SafeChildWatcher() - - -class FastChildWatcherTests (ChildWatcherTestsMixin, test_utils.TestCase): - def create_watcher(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - return asyncio.FastChildWatcher() - - -class PolicyTests(unittest.TestCase): - - def create_policy(self): - return asyncio.DefaultEventLoopPolicy() - - @mock.patch('asyncio.unix_events.can_use_pidfd') - def test_get_default_child_watcher(self, m_can_use_pidfd): - m_can_use_pidfd.return_value = False - policy = self.create_policy() - self.assertIsNone(policy._watcher) - with self.assertWarns(DeprecationWarning): - watcher = policy.get_child_watcher() - self.assertIsInstance(watcher, asyncio.ThreadedChildWatcher) - - self.assertIs(policy._watcher, watcher) - with self.assertWarns(DeprecationWarning): - self.assertIs(watcher, policy.get_child_watcher()) - - m_can_use_pidfd.return_value = True - policy = self.create_policy() - self.assertIsNone(policy._watcher) - with self.assertWarns(DeprecationWarning): - watcher = policy.get_child_watcher() - self.assertIsInstance(watcher, asyncio.PidfdChildWatcher) - - self.assertIs(policy._watcher, watcher) - with self.assertWarns(DeprecationWarning): - self.assertIs(watcher, policy.get_child_watcher()) - - def test_get_child_watcher_after_set(self): - policy = self.create_policy() - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - watcher = asyncio.FastChildWatcher() - policy.set_child_watcher(watcher) - - self.assertIs(policy._watcher, watcher) - with self.assertWarns(DeprecationWarning): - self.assertIs(watcher, policy.get_child_watcher()) - - def test_get_child_watcher_thread(self): - - def f(): - policy.set_event_loop(policy.new_event_loop()) - - self.assertIsInstance(policy.get_event_loop(), - asyncio.AbstractEventLoop) - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - watcher = policy.get_child_watcher() - - self.assertIsInstance(watcher, asyncio.SafeChildWatcher) - self.assertIsNone(watcher._loop) - - policy.get_event_loop().close() - - policy = self.create_policy() - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - policy.set_child_watcher(asyncio.SafeChildWatcher()) - - th = threading.Thread(target=f) - th.start() - th.join() - - def test_child_watcher_replace_mainloop_existing(self): - policy = self.create_policy() - loop = policy.new_event_loop() - policy.set_event_loop(loop) - - # Explicitly setup SafeChildWatcher, - # default ThreadedChildWatcher has no _loop property - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - watcher = asyncio.SafeChildWatcher() - policy.set_child_watcher(watcher) - watcher.attach_loop(loop) - - self.assertIs(watcher._loop, loop) - - new_loop = policy.new_event_loop() - policy.set_event_loop(new_loop) - - self.assertIs(watcher._loop, new_loop) - - policy.set_event_loop(None) - - self.assertIs(watcher._loop, None) - - loop.close() - new_loop.close() - - class TestFunctional(unittest.TestCase): def setUp(self): @@ -1874,11 +1180,46 @@ async def runner(): @support.requires_fork() -class TestFork(unittest.IsolatedAsyncioTestCase): +class TestFork(unittest.TestCase): + + def test_fork_not_share_current_task(self): + loop = object() + task = object() + asyncio._set_running_loop(loop) + self.addCleanup(asyncio._set_running_loop, None) + asyncio.tasks._enter_task(loop, task) + self.addCleanup(asyncio.tasks._leave_task, loop, task) + self.assertIs(asyncio.current_task(), task) + r, w = os.pipe() + self.addCleanup(os.close, r) + self.addCleanup(os.close, w) + pid = os.fork() + if pid == 0: + # child + try: + asyncio._set_running_loop(loop) + current_task = asyncio.current_task() + if current_task is None: + os.write(w, b'NO TASK') + else: + os.write(w, b'TASK:' + str(id(current_task)).encode()) + except BaseException as e: + os.write(w, b'ERROR:' + ascii(e).encode()) + finally: + asyncio._set_running_loop(None) + os._exit(0) + else: + # parent + result = os.read(r, 100) + self.assertEqual(result, b'NO TASK') + wait_process(pid, exitcode=0) - async def test_fork_not_share_event_loop(self): + def test_fork_not_share_event_loop(self): # The forked process should not share the event loop with the parent - loop = asyncio.get_running_loop() + loop = object() + asyncio._set_running_loop(loop) + self.assertIs(asyncio.get_running_loop(), loop) + self.addCleanup(asyncio._set_running_loop, None) r, w = os.pipe() self.addCleanup(os.close, r) self.addCleanup(os.close, w) @@ -1886,8 +1227,7 @@ async def test_fork_not_share_event_loop(self): if pid == 0: # child try: - with self.assertWarns(DeprecationWarning): - loop = asyncio.get_event_loop_policy().get_event_loop() + loop = asyncio.get_event_loop() os.write(w, b'LOOP:' + str(id(loop)).encode()) except RuntimeError: os.write(w, b'NO LOOP') @@ -1898,8 +1238,7 @@ async def test_fork_not_share_event_loop(self): else: # parent result = os.read(r, 100) - self.assertEqual(result[:5], b'LOOP:', result) - self.assertNotEqual(int(result[5:]), id(loop)) + self.assertEqual(result, b'NO LOOP') wait_process(pid, exitcode=0) @hashlib_helper.requires_hashdigest('md5') diff --git a/Lib/test/test_asyncio/test_waitfor.py b/Lib/test/test_asyncio/test_waitfor.py index 11a8eeeab37..dedc6bf69d7 100644 --- a/Lib/test/test_asyncio/test_waitfor.py +++ b/Lib/test/test_asyncio/test_waitfor.py @@ -5,7 +5,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) # The following value can be used as a very small timeout: diff --git a/Lib/test/test_asyncio/test_windows_events.py b/Lib/test/test_asyncio/test_windows_events.py index 0c128c599ba..0af3368627a 100644 --- a/Lib/test/test_asyncio/test_windows_events.py +++ b/Lib/test/test_asyncio/test_windows_events.py @@ -19,7 +19,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class UpperProto(asyncio.Protocol): @@ -328,17 +328,18 @@ class WinPolicyTests(WindowsEventsTestCase): def test_selector_win_policy(self): async def main(): - self.assertIsInstance( - asyncio.get_running_loop(), - asyncio.SelectorEventLoop) + self.assertIsInstance(asyncio.get_running_loop(), asyncio.SelectorEventLoop) - old_policy = asyncio.get_event_loop_policy() + old_policy = asyncio.events._get_event_loop_policy() try: - asyncio.set_event_loop_policy( - asyncio.WindowsSelectorEventLoopPolicy()) + with self.assertWarnsRegex( + DeprecationWarning, + "'asyncio.WindowsSelectorEventLoopPolicy' is deprecated", + ): + asyncio.events._set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) asyncio.run(main()) finally: - asyncio.set_event_loop_policy(old_policy) + asyncio.events._set_event_loop_policy(old_policy) def test_proactor_win_policy(self): async def main(): @@ -346,13 +347,16 @@ async def main(): asyncio.get_running_loop(), asyncio.ProactorEventLoop) - old_policy = asyncio.get_event_loop_policy() + old_policy = asyncio.events._get_event_loop_policy() try: - asyncio.set_event_loop_policy( - asyncio.WindowsProactorEventLoopPolicy()) + with self.assertWarnsRegex( + DeprecationWarning, + "'asyncio.WindowsProactorEventLoopPolicy' is deprecated", + ): + asyncio.events._set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy()) asyncio.run(main()) finally: - asyncio.set_event_loop_policy(old_policy) + asyncio.events._set_event_loop_policy(old_policy) if __name__ == '__main__': diff --git a/Lib/test/test_asyncio/test_windows_utils.py b/Lib/test/test_asyncio/test_windows_utils.py index eafa5be3829..97f078ff911 100644 --- a/Lib/test/test_asyncio/test_windows_utils.py +++ b/Lib/test/test_asyncio/test_windows_utils.py @@ -16,7 +16,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class PipeTests(unittest.TestCase): @@ -121,8 +121,8 @@ def test_popen(self): self.assertGreater(len(out), 0) self.assertGreater(len(err), 0) # allow for partial reads... - self.assertTrue(msg.upper().rstrip().startswith(out)) - self.assertTrue(b"stderr".startswith(err)) + self.assertStartsWith(msg.upper().rstrip(), out) + self.assertStartsWith(b"stderr", err) # The context manager calls wait() and closes resources with p: diff --git a/Lib/test/test_asyncio/utils.py b/Lib/test/test_asyncio/utils.py index d9a2939be13..a480e16e81b 100644 --- a/Lib/test/test_asyncio/utils.py +++ b/Lib/test/test_asyncio/utils.py @@ -14,7 +14,7 @@ import threading import unittest import weakref -import warnings +from ast import literal_eval from unittest import mock from http.server import HTTPServer @@ -55,24 +55,8 @@ def data_file(*filename): ONLYKEY = data_file('certdata', 'ssl_key.pem') SIGNED_CERTFILE = data_file('certdata', 'keycert3.pem') SIGNING_CA = data_file('certdata', 'pycacert.pem') -PEERCERT = { - 'OCSP': ('http://testca.pythontest.net/testca/ocsp/',), - 'caIssuers': ('http://testca.pythontest.net/testca/pycacert.cer',), - 'crlDistributionPoints': ('http://testca.pythontest.net/testca/revocation.crl',), - 'issuer': ((('countryName', 'XY'),), - (('organizationName', 'Python Software Foundation CA'),), - (('commonName', 'our-ca-server'),)), - 'notAfter': 'Oct 28 14:23:16 2037 GMT', - 'notBefore': 'Aug 29 14:23:16 2018 GMT', - 'serialNumber': 'CB2D80995A69525C', - 'subject': ((('countryName', 'XY'),), - (('localityName', 'Castle Anthrax'),), - (('organizationName', 'Python Software Foundation'),), - (('commonName', 'localhost'),)), - 'subjectAltName': (('DNS', 'localhost'),), - 'version': 3 -} - +with open(data_file('certdata', 'keycert3.pem.reference')) as file: + PEERCERT = literal_eval(file.read()) def simple_server_sslcontext(): server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) @@ -551,25 +535,6 @@ def close_loop(loop): loop._default_executor.shutdown(wait=True) loop.close() - policy = support.maybe_get_event_loop_policy() - if policy is not None: - try: - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - watcher = policy.get_child_watcher() - except NotImplementedError: - # watcher is not implemented by EventLoopPolicy, e.g. Windows - pass - else: - if isinstance(watcher, asyncio.ThreadedChildWatcher): - # Wait for subprocess to finish, but not forever - for thread in list(watcher._threads.values()): - thread.join(timeout=support.SHORT_TIMEOUT) - if thread.is_alive(): - raise RuntimeError(f"thread {thread} still alive: " - "subprocess still running") - - def set_event_loop(self, loop, *, cleanup=True): if loop is None: raise AssertionError('loop is None') @@ -636,3 +601,9 @@ def func(): await asyncio.sleep(0) if exc is not None: raise exc + + +if sys.platform == 'win32': + DefaultEventLoopPolicy = asyncio.windows_events._DefaultEventLoopPolicy +else: + DefaultEventLoopPolicy = asyncio.unix_events._DefaultEventLoopPolicy diff --git a/Lib/test/test_atexit.py b/Lib/test/test_atexit.py index 913b7556be8..6f57ee06879 100644 --- a/Lib/test/test_atexit.py +++ b/Lib/test/test_atexit.py @@ -1,10 +1,12 @@ import atexit import os +import subprocess import textwrap import unittest +from test.support import os_helper from test import support -from test.support import script_helper - +from test.support import SuppressCrashReport, script_helper +from test.support import threading_helper class GeneralTest(unittest.TestCase): def test_general(self): @@ -46,6 +48,40 @@ def test_atexit_instances(self): self.assertEqual(res.out.decode().splitlines(), ["atexit2", "atexit1"]) self.assertFalse(res.err) + @unittest.skip("TODO: RUSTPYTHON; Flakey on CI") + @threading_helper.requires_working_threading() + @support.requires_resource("cpu") + @unittest.skipUnless(support.Py_GIL_DISABLED, "only meaningful without the GIL") + def test_atexit_thread_safety(self): + # GH-126907: atexit was not thread safe on the free-threaded build + source = """ + from threading import Thread + + def dummy(): + pass + + + def thready(): + for _ in range(100): + atexit.register(dummy) + atexit._clear() + atexit.register(dummy) + atexit.unregister(dummy) + atexit._run_exitfuncs() + + + threads = [Thread(target=thready) for _ in range(10)] + for thread in threads: + thread.start() + + for thread in threads: + thread.join() + """ + + # atexit._clear() has some evil side effects, and we don't + # want them to affect the rest of the tests. + script_helper.assert_python_ok("-c", textwrap.dedent(source)) + @support.cpython_only class SubinterpreterTest(unittest.TestCase): @@ -100,6 +136,37 @@ def callback(): self.assertEqual(os.read(r, len(expected)), expected) os.close(r) + # Python built with Py_TRACE_REFS fail with a fatal error in + # _PyRefchain_Trace() on memory allocation error. + @unittest.skipIf(support.Py_TRACE_REFS, 'cannot test Py_TRACE_REFS build') + def test_atexit_with_low_memory(self): + # gh-140080: Test that setting low memory after registering an atexit + # callback doesn't cause an infinite loop during finalization. + code = textwrap.dedent(""" + import atexit + import _testcapi + + def callback(): + print("hello") + + atexit.register(callback) + # Simulate low memory condition + _testcapi.set_nomemory(0) + """) + + with os_helper.temp_dir() as temp_dir: + script = script_helper.make_script(temp_dir, 'test_atexit_script', code) + with SuppressCrashReport(): + with script_helper.spawn_python(script, + stderr=subprocess.PIPE) as proc: + proc.wait() + stdout = proc.stdout.read() + stderr = proc.stderr.read() + + self.assertIn(proc.returncode, (0, 1)) + self.assertNotIn(b"hello", stdout) + self.assertIn(b"MemoryError", stderr) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_baseexception.py b/Lib/test/test_baseexception.py index 63bf538aa53..5870dc7f9da 100644 --- a/Lib/test/test_baseexception.py +++ b/Lib/test/test_baseexception.py @@ -10,13 +10,11 @@ class ExceptionClassTests(unittest.TestCase): inheritance hierarchy)""" def test_builtins_new_style(self): - self.assertTrue(issubclass(Exception, object)) + self.assertIsSubclass(Exception, object) def verify_instance_interface(self, ins): for attr in ("args", "__str__", "__repr__"): - self.assertTrue(hasattr(ins, attr), - "%s missing %s attribute" % - (ins.__class__.__name__, attr)) + self.assertHasAttr(ins, attr) def test_inheritance(self): # Make sure the inheritance hierarchy matches the documentation @@ -65,7 +63,7 @@ def test_inheritance(self): elif last_depth > depth: while superclasses[-1][0] >= depth: superclasses.pop() - self.assertTrue(issubclass(exc, superclasses[-1][1]), + self.assertIsSubclass(exc, superclasses[-1][1], "%s is not a subclass of %s" % (exc.__name__, superclasses[-1][1].__name__)) try: # Some exceptions require arguments; just skip them diff --git a/Lib/test/test_bdb.py b/Lib/test/test_bdb.py index c41fb763a16..f15dae13eb3 100644 --- a/Lib/test/test_bdb.py +++ b/Lib/test/test_bdb.py @@ -590,7 +590,6 @@ def fail(self, msg=None): class StateTestCase(BaseTestCase): """Test the step, next, return, until and quit 'set_' methods.""" - @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_step(self): self.expect_set = [ ('line', 2, 'tfunc_main'), ('step', ), @@ -601,7 +600,6 @@ def test_step(self): with TracerRun(self) as tracer: tracer.runcall(tfunc_main) - @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_step_next_on_last_statement(self): for set_type in ('step', 'next'): with self.subTest(set_type=set_type): @@ -616,7 +614,6 @@ def test_step_next_on_last_statement(self): with TracerRun(self) as tracer: tracer.runcall(tfunc_main) - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: All paired tuples have not been processed, the last one was number 1 [('next',), ('quit',)] def test_stepinstr(self): self.expect_set = [ ('line', 2, 'tfunc_main'), ('stepinstr', ), @@ -626,7 +623,6 @@ def test_stepinstr(self): with TracerRun(self) as tracer: tracer.runcall(tfunc_main) - @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_next(self): self.expect_set = [ ('line', 2, 'tfunc_main'), ('step', ), @@ -638,7 +634,6 @@ def test_next(self): with TracerRun(self) as tracer: tracer.runcall(tfunc_main) - @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_next_over_import(self): code = """ def main(): @@ -653,7 +648,6 @@ def main(): with TracerRun(self) as tracer: tracer.runcall(tfunc_import) - @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_next_on_plain_statement(self): # Check that set_next() is equivalent to set_step() on a plain # statement. @@ -666,7 +660,6 @@ def test_next_on_plain_statement(self): with TracerRun(self) as tracer: tracer.runcall(tfunc_main) - @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_next_in_caller_frame(self): # Check that set_next() in the caller frame causes the tracer # to stop next in the caller frame. @@ -680,7 +673,6 @@ def test_next_in_caller_frame(self): with TracerRun(self) as tracer: tracer.runcall(tfunc_main) - @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_return(self): self.expect_set = [ ('line', 2, 'tfunc_main'), ('step', ), @@ -693,7 +685,6 @@ def test_return(self): with TracerRun(self) as tracer: tracer.runcall(tfunc_main) - @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_return_in_caller_frame(self): self.expect_set = [ ('line', 2, 'tfunc_main'), ('step', ), @@ -705,7 +696,6 @@ def test_return_in_caller_frame(self): with TracerRun(self) as tracer: tracer.runcall(tfunc_main) - @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_until(self): self.expect_set = [ ('line', 2, 'tfunc_main'), ('step', ), @@ -717,7 +707,6 @@ def test_until(self): with TracerRun(self) as tracer: tracer.runcall(tfunc_main) - @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_until_with_too_large_count(self): self.expect_set = [ ('line', 2, 'tfunc_main'), break_in_func('tfunc_first'), @@ -728,7 +717,6 @@ def test_until_with_too_large_count(self): with TracerRun(self) as tracer: tracer.runcall(tfunc_main) - @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_until_in_caller_frame(self): self.expect_set = [ ('line', 2, 'tfunc_main'), ('step', ), @@ -740,7 +728,6 @@ def test_until_in_caller_frame(self): with TracerRun(self) as tracer: tracer.runcall(tfunc_main) - @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs @patch_list(sys.meta_path) def test_skip(self): # Check that tracing is skipped over the import statement in @@ -774,7 +761,6 @@ def test_skip_with_no_name_module(self): bdb = Bdb(skip=['anything*']) self.assertIs(bdb.is_skipped_module(None), False) - @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_down(self): # Check that set_down() raises BdbError at the newest frame. self.expect_set = [ @@ -783,7 +769,6 @@ def test_down(self): with TracerRun(self) as tracer: self.assertRaises(BdbError, tracer.runcall, tfunc_main) - @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_up(self): self.expect_set = [ ('line', 2, 'tfunc_main'), ('step', ), @@ -797,7 +782,6 @@ def test_up(self): class BreakpointTestCase(BaseTestCase): """Test the breakpoint set method.""" - @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_bp_on_non_existent_module(self): self.expect_set = [ ('line', 2, 'tfunc_import'), ('break', ('/non/existent/module.py', 1)) @@ -805,7 +789,6 @@ def test_bp_on_non_existent_module(self): with TracerRun(self) as tracer: self.assertRaises(BdbError, tracer.runcall, tfunc_import) - @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_bp_after_last_statement(self): code = """ def main(): @@ -819,7 +802,6 @@ def main(): with TracerRun(self) as tracer: self.assertRaises(BdbError, tracer.runcall, tfunc_import) - @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_temporary_bp(self): code = """ def func(): @@ -843,7 +825,6 @@ def main(): with TracerRun(self) as tracer: tracer.runcall(tfunc_import) - @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_disabled_temporary_bp(self): code = """ def func(): @@ -872,7 +853,6 @@ def main(): with TracerRun(self) as tracer: tracer.runcall(tfunc_import) - @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_bp_condition(self): code = """ def func(a): @@ -893,7 +873,6 @@ def main(): with TracerRun(self) as tracer: tracer.runcall(tfunc_import) - @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_bp_exception_on_condition_evaluation(self): code = """ def func(a): @@ -913,7 +892,6 @@ def main(): with TracerRun(self) as tracer: tracer.runcall(tfunc_import) - @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_bp_ignore_count(self): code = """ def func(): @@ -935,7 +913,6 @@ def main(): with TracerRun(self) as tracer: tracer.runcall(tfunc_import) - @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_ignore_count_on_disabled_bp(self): code = """ def func(): @@ -963,7 +940,6 @@ def main(): with TracerRun(self) as tracer: tracer.runcall(tfunc_import) - @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_clear_two_bp_on_same_line(self): code = """ def func(): @@ -989,7 +965,6 @@ def main(): with TracerRun(self) as tracer: tracer.runcall(tfunc_import) - @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_clear_at_no_bp(self): self.expect_set = [ ('line', 2, 'tfunc_import'), ('clear', (__file__, 1)) @@ -1043,7 +1018,6 @@ def test_load_bps_from_previous_Bdb_instance(self): class RunTestCase(BaseTestCase): """Test run, runeval and set_trace.""" - @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_run_step(self): # Check that the bdb 'run' method stops at the first line event. code = """ @@ -1056,7 +1030,6 @@ def test_run_step(self): with TracerRun(self) as tracer: tracer.run(compile(textwrap.dedent(code), '', 'exec')) - @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_runeval_step(self): # Test bdb 'runeval'. code = """ @@ -1073,13 +1046,13 @@ def main(): ('return', 1, ''), ('quit', ), ] import test_module_for_bdb + ns = {'test_module_for_bdb': test_module_for_bdb} with TracerRun(self) as tracer: - tracer.runeval('test_module_for_bdb.main()', globals(), locals()) + tracer.runeval('test_module_for_bdb.main()', ns, ns) class IssuesTestCase(BaseTestCase): """Test fixed bdb issues.""" - @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_step_at_return_with_no_trace_in_caller(self): # Issue #13183. # Check that the tracer does step into the caller frame when the @@ -1110,7 +1083,6 @@ def func(): with TracerRun(self) as tracer: tracer.runcall(tfunc_import) - @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_next_until_return_in_generator(self): # Issue #16596. # Check that set_next(), set_until() and set_return() do not treat the @@ -1152,7 +1124,6 @@ def main(): with TracerRun(self) as tracer: tracer.runcall(tfunc_import) - @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_next_command_in_generator_for_loop(self): # Issue #16596. code = """ @@ -1184,7 +1155,6 @@ def main(): with TracerRun(self) as tracer: tracer.runcall(tfunc_import) - @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_next_command_in_generator_with_subiterator(self): # Issue #16596. code = """ @@ -1216,7 +1186,6 @@ def main(): with TracerRun(self) as tracer: tracer.runcall(tfunc_import) - @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_return_command_in_generator_with_subiterator(self): # Issue #16596. code = """ @@ -1248,7 +1217,6 @@ def main(): with TracerRun(self) as tracer: tracer.runcall(tfunc_import) - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: All paired tuples have not been processed, the last one was number 1 [('next',)] def test_next_to_botframe(self): # gh-125422 # Check that next command won't go to the bottom frame. diff --git a/Lib/test/test_bigmem.py b/Lib/test/test_bigmem.py index aaa9972bc45..8f528812e35 100644 --- a/Lib/test/test_bigmem.py +++ b/Lib/test/test_bigmem.py @@ -638,8 +638,6 @@ def test_encode_utf7(self, size): except MemoryError: pass # acceptable on 32-bit - # TODO: RUSTPYTHON - @unittest.expectedFailure @bigmemtest(size=_4G // 4 + 5, memuse=ascii_char_size + ucs4_char_size + 4) def test_encode_utf32(self, size): try: diff --git a/Lib/test/test_binascii.py b/Lib/test/test_binascii.py index cf11ffce7f1..fa027710489 100644 --- a/Lib/test/test_binascii.py +++ b/Lib/test/test_binascii.py @@ -38,13 +38,13 @@ def assertConversion(self, original, converted, restored, **kwargs): def test_exceptions(self): # Check module exceptions - self.assertTrue(issubclass(binascii.Error, Exception)) - self.assertTrue(issubclass(binascii.Incomplete, Exception)) + self.assertIsSubclass(binascii.Error, Exception) + self.assertIsSubclass(binascii.Incomplete, Exception) def test_functions(self): # Check presence of all functions for name in all_functions: - self.assertTrue(hasattr(getattr(binascii, name), '__call__')) + self.assertHasAttr(getattr(binascii, name), '__call__') self.assertRaises(TypeError, getattr(binascii, name)) def test_returned_value(self): @@ -117,7 +117,7 @@ def addnoise(line): # empty strings. TBD: shouldn't it raise an exception instead ? self.assertEqual(binascii.a2b_base64(self.type2test(fillers)), b'') - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_base64_strict_mode(self): # Test base64 with strict mode on def _assertRegexTemplate(assert_regex: str, data: bytes, non_strict_mode_expected_result: bytes): diff --git a/Lib/test/test_binop.py b/Lib/test/test_binop.py index 299af09c498..b224c3d4e60 100644 --- a/Lib/test/test_binop.py +++ b/Lib/test/test_binop.py @@ -383,7 +383,7 @@ def test_comparison_orders(self): self.assertEqual(op_sequence(le, B, C), ['C.__ge__', 'B.__le__']) self.assertEqual(op_sequence(le, C, B), ['C.__le__', 'B.__ge__']) - self.assertTrue(issubclass(V, B)) + self.assertIsSubclass(V, B) self.assertEqual(op_sequence(eq, B, V), ['B.__eq__', 'V.__eq__']) self.assertEqual(op_sequence(le, B, V), ['B.__le__', 'V.__ge__']) diff --git a/Lib/test/test_bool.py b/Lib/test/test_bool.py index 34ecb45f161..dcdf7bdce03 100644 --- a/Lib/test/test_bool.py +++ b/Lib/test/test_bool.py @@ -383,6 +383,10 @@ def __len__(self): __bool__ = None self.assertRaises(TypeError, bool, B()) + class C: + __len__ = None + self.assertRaises(TypeError, bool, C()) + def test_real_and_imag(self): self.assertEqual(True.real, 1) self.assertEqual(True.imag, 0) diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index 468c6ea9def..bc09329e6de 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -17,12 +17,14 @@ import unittest from test import support from test.support import os_helper +import inspect from itertools import permutations, product from random import randrange, sample, choice import warnings import sys, array, io, os from decimal import Decimal from fractions import Fraction +from test.support import warnings_helper try: from _testbuffer import * @@ -64,7 +66,7 @@ '?':0, 'c':0, 'b':0, 'B':0, 'h':0, 'H':0, 'i':0, 'I':0, 'l':0, 'L':0, 'n':0, 'N':0, - 'f':0, 'd':0, 'P':0 + 'e':0, 'f':0, 'd':0, 'P':0 } # NumPy does not have 'n' or 'N': @@ -89,7 +91,8 @@ 'i':(-(1<<31), 1<<31), 'I':(0, 1<<32), 'l':(-(1<<31), 1<<31), 'L':(0, 1<<32), 'q':(-(1<<63), 1<<63), 'Q':(0, 1<<64), - 'f':(-(1<<63), 1<<63), 'd':(-(1<<1023), 1<<1023) + 'e':(-65519, 65520), 'f':(-(1<<63), 1<<63), + 'd':(-(1<<1023), 1<<1023) } def native_type_range(fmt): @@ -98,6 +101,8 @@ def native_type_range(fmt): lh = (0, 256) elif fmt == '?': lh = (0, 2) + elif fmt == 'e': + lh = (-65519, 65520) elif fmt == 'f': lh = (-(1<<63), 1<<63) elif fmt == 'd': @@ -125,7 +130,10 @@ def native_type_range(fmt): for fmt in fmtdict['@']: fmtdict['@'][fmt] = native_type_range(fmt) +# Format codes supported by the memoryview object MEMORYVIEW = NATIVE.copy() + +# Format codes supported by array.array ARRAY = NATIVE.copy() for k in NATIVE: if not k in "bBhHiIlLfd": @@ -160,11 +168,11 @@ def randrange_fmt(mode, char, obj): if char == 'c': x = bytes([x]) if obj == 'numpy' and x == b'\x00': - # http://projects.scipy.org/numpy/ticket/1925 + # https://github.com/numpy/numpy/issues/2518 x = b'\x01' if char == '?': x = bool(x) - if char == 'f' or char == 'd': + if char in 'efd': x = struct.pack(char, x) x = struct.unpack(char, x)[0] return x @@ -959,8 +967,10 @@ def check_memoryview(m, expected_readonly=readonly): self.assertEqual(m.strides, tuple(strides)) self.assertEqual(m.suboffsets, tuple(suboffsets)) - n = 1 if ndim == 0 else len(lst) - self.assertEqual(len(m), n) + if ndim == 0: + self.assertRaises(TypeError, len, m) + else: + self.assertEqual(len(m), len(lst)) rep = result.tolist() if fmt else result.tobytes() self.assertEqual(rep, lst) @@ -1019,6 +1029,7 @@ def match(req, flag): ndim=ndim, shape=shape, strides=strides, lst=lst, sliced=sliced) + @support.requires_resource('cpu') def test_ndarray_getbuf(self): requests = ( # distinct flags @@ -1907,7 +1918,7 @@ def test_ndarray_random(self): if numpy_array: shape = t[3] if 0 in shape: - continue # http://projects.scipy.org/numpy/ticket/1910 + continue # https://github.com/numpy/numpy/issues/2503 z = numpy_array_from_structure(items, fmt, t) self.verify(x, obj=None, itemsize=z.itemsize, fmt=fmt, readonly=False, @@ -1939,7 +1950,7 @@ def test_ndarray_random_invalid(self): except Exception as e: numpy_err = e.__class__ - if 0: # http://projects.scipy.org/numpy/ticket/1910 + if 0: # https://github.com/numpy/numpy/issues/2503 self.assertTrue(numpy_err) def test_ndarray_random_slice_assign(self): @@ -1985,7 +1996,7 @@ def test_ndarray_random_slice_assign(self): if numpy_array: if 0 in lshape or 0 in rshape: - continue # http://projects.scipy.org/numpy/ticket/1910 + continue # https://github.com/numpy/numpy/issues/2503 zl = numpy_array_from_structure(litems, fmt, tl) zr = numpy_array_from_structure(ritems, fmt, tr) @@ -2246,7 +2257,7 @@ def test_py_buffer_to_contiguous(self): ### ### Fortran output: ### --------------- - ### >>> fortran_buf = nd.tostring(order='F') + ### >>> fortran_buf = nd.tobytes(order='F') ### >>> fortran_buf ### b'\x00\x04\x08\x01\x05\t\x02\x06\n\x03\x07\x0b' ### @@ -2289,7 +2300,7 @@ def test_py_buffer_to_contiguous(self): self.assertEqual(memoryview(y), memoryview(nd)) if numpy_array: - self.assertEqual(b, na.tostring(order='C')) + self.assertEqual(b, na.tobytes(order='C')) # 'F' request if f == 0: # 'C' to 'F' @@ -2312,7 +2323,7 @@ def test_py_buffer_to_contiguous(self): self.assertEqual(memoryview(y), memoryview(nd)) if numpy_array: - self.assertEqual(b, na.tostring(order='F')) + self.assertEqual(b, na.tobytes(order='F')) # 'A' request if f == ND_FORTRAN: @@ -2336,7 +2347,7 @@ def test_py_buffer_to_contiguous(self): self.assertEqual(memoryview(y), memoryview(nd)) if numpy_array: - self.assertEqual(b, na.tostring(order='A')) + self.assertEqual(b, na.tobytes(order='A')) # multi-dimensional, non-contiguous input nd = ndarray(list(range(12)), shape=[3, 4], flags=ND_WRITABLE|ND_PIL) @@ -2750,6 +2761,7 @@ def iter_roundtrip(ex, m, items, fmt): m = memoryview(ex) iter_roundtrip(ex, m, items, fmt) + @support.requires_resource('cpu') def test_memoryview_cast_1D_ND(self): # Cast between C-contiguous buffers. At least one buffer must # be 1D, at least one format must be 'c', 'b' or 'B'. @@ -2867,11 +2879,11 @@ def test_memoryview_tolist(self): def test_memoryview_repr(self): m = memoryview(bytearray(9)) r = m.__repr__() - self.assertTrue(r.startswith("l:x:>l:y:}" @@ -3227,6 +3233,15 @@ class BEPoint(ctypes.BigEndianStructure): self.assertNotEqual(point, a) self.assertRaises(NotImplementedError, a.tolist) + @warnings_helper.ignore_warnings(category=DeprecationWarning) # gh-80480 array('u') + def test_memoryview_compare_special_cases_deprecated_u_type_code(self): + + # Depends on issue #15625: the struct module does not understand 'u'. + a = array.array('u', 'xyz') + v = memoryview(a) + self.assertNotEqual(a, v) + self.assertNotEqual(v, a) + def test_memoryview_compare_ndim_zero(self): nd1 = ndarray(1729, shape=[], format='@L') @@ -3895,6 +3910,8 @@ def test_memoryview_check_released(self): self.assertRaises(ValueError, memoryview, m) # memoryview.cast() self.assertRaises(ValueError, m.cast, 'c') + # memoryview.__iter__() + self.assertRaises(ValueError, m.__iter__) # getbuffer() self.assertRaises(ValueError, ndarray, m) # memoryview.tolist() @@ -4422,6 +4439,14 @@ def test_issue_7385(self): x = ndarray([1,2,3], shape=[3], flags=ND_GETBUF_FAIL) self.assertRaises(BufferError, memoryview, x) + def test_bytearray_release_buffer_read_flag(self): + # See https://github.com/python/cpython/issues/126980 + obj = bytearray(b'abc') + with self.assertRaises(SystemError): + obj.__buffer__(inspect.BufferFlags.READ) + with self.assertRaises(SystemError): + obj.__buffer__(inspect.BufferFlags.WRITE) + @support.cpython_only def test_pybuffer_size_from_format(self): # basic tests @@ -4429,6 +4454,383 @@ def test_pybuffer_size_from_format(self): self.assertEqual(_testcapi.PyBuffer_SizeFromFormat(format), struct.calcsize(format)) + @support.cpython_only + def test_flags_overflow(self): + # gh-126594: Check for integer overlow on large flags + try: + from _testcapi import INT_MIN, INT_MAX + except ImportError: + INT_MIN = -(2 ** 31) + INT_MAX = 2 ** 31 - 1 + + obj = b'abc' + for flags in (INT_MIN - 1, INT_MAX + 1): + with self.subTest(flags=flags): + with self.assertRaises(OverflowError): + obj.__buffer__(flags) + + +class TestPythonBufferProtocol(unittest.TestCase): + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_basic(self): + class MyBuffer: + def __buffer__(self, flags): + return memoryview(b"hello") + + mv = memoryview(MyBuffer()) + self.assertEqual(mv.tobytes(), b"hello") + self.assertEqual(bytes(MyBuffer()), b"hello") + + def test_bad_buffer_method(self): + class MustReturnMV: + def __buffer__(self, flags): + return 42 + + self.assertRaises(TypeError, memoryview, MustReturnMV()) + + class NoBytesEither: + def __buffer__(self, flags): + return b"hello" + + self.assertRaises(TypeError, memoryview, NoBytesEither()) + + class WrongArity: + def __buffer__(self): + return memoryview(b"hello") + + self.assertRaises(TypeError, memoryview, WrongArity()) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_release_buffer(self): + class WhatToRelease: + def __init__(self): + self.held = False + self.ba = bytearray(b"hello") + + def __buffer__(self, flags): + if self.held: + raise TypeError("already held") + self.held = True + return memoryview(self.ba) + + def __release_buffer__(self, buffer): + self.held = False + + wr = WhatToRelease() + self.assertFalse(wr.held) + with memoryview(wr) as mv: + self.assertTrue(wr.held) + self.assertEqual(mv.tobytes(), b"hello") + self.assertFalse(wr.held) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_same_buffer_returned(self): + class WhatToRelease: + def __init__(self): + self.held = False + self.ba = bytearray(b"hello") + self.created_mv = None + + def __buffer__(self, flags): + if self.held: + raise TypeError("already held") + self.held = True + self.created_mv = memoryview(self.ba) + return self.created_mv + + def __release_buffer__(self, buffer): + assert buffer is self.created_mv + self.held = False + + wr = WhatToRelease() + self.assertFalse(wr.held) + with memoryview(wr) as mv: + self.assertTrue(wr.held) + self.assertEqual(mv.tobytes(), b"hello") + self.assertFalse(wr.held) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_buffer_flags(self): + class PossiblyMutable: + def __init__(self, data, mutable) -> None: + self._data = bytearray(data) + self._mutable = mutable + + def __buffer__(self, flags): + if flags & inspect.BufferFlags.WRITABLE: + if not self._mutable: + raise RuntimeError("not mutable") + return memoryview(self._data) + else: + return memoryview(bytes(self._data)) + + mutable = PossiblyMutable(b"hello", True) + immutable = PossiblyMutable(b"hello", False) + with memoryview._from_flags(mutable, inspect.BufferFlags.WRITABLE) as mv: + self.assertEqual(mv.tobytes(), b"hello") + mv[0] = ord(b'x') + self.assertEqual(mv.tobytes(), b"xello") + with memoryview._from_flags(mutable, inspect.BufferFlags.SIMPLE) as mv: + self.assertEqual(mv.tobytes(), b"xello") + with self.assertRaises(TypeError): + mv[0] = ord(b'h') + self.assertEqual(mv.tobytes(), b"xello") + with memoryview._from_flags(immutable, inspect.BufferFlags.SIMPLE) as mv: + self.assertEqual(mv.tobytes(), b"hello") + with self.assertRaises(TypeError): + mv[0] = ord(b'x') + self.assertEqual(mv.tobytes(), b"hello") + + with self.assertRaises(RuntimeError): + memoryview._from_flags(immutable, inspect.BufferFlags.WRITABLE) + with memoryview(immutable) as mv: + self.assertEqual(mv.tobytes(), b"hello") + with self.assertRaises(TypeError): + mv[0] = ord(b'x') + self.assertEqual(mv.tobytes(), b"hello") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_call_builtins(self): + ba = bytearray(b"hello") + mv = ba.__buffer__(0) + self.assertEqual(mv.tobytes(), b"hello") + ba.__release_buffer__(mv) + with self.assertRaises(OverflowError): + ba.__buffer__(sys.maxsize + 1) + + @unittest.skipIf(_testcapi is None, "requires _testcapi") + def test_c_buffer(self): + buf = _testcapi.testBuf() + self.assertEqual(buf.references, 0) + mv = buf.__buffer__(0) + self.assertIsInstance(mv, memoryview) + self.assertEqual(mv.tobytes(), b"test") + self.assertEqual(buf.references, 1) + buf.__release_buffer__(mv) + self.assertEqual(buf.references, 0) + with self.assertRaises(ValueError): + mv.tobytes() + # Calling it again doesn't cause issues + with self.assertRaises(ValueError): + buf.__release_buffer__(mv) + self.assertEqual(buf.references, 0) + + @unittest.skipIf(_testcapi is None, "requires _testcapi") + def test_c_buffer_invalid_flags(self): + buf = _testcapi.testBuf() + self.assertRaises(SystemError, buf.__buffer__, PyBUF_READ) + self.assertRaises(SystemError, buf.__buffer__, PyBUF_WRITE) + + @unittest.skipIf(_testcapi is None, "requires _testcapi") + def test_c_fill_buffer_invalid_flags(self): + # PyBuffer_FillInfo + source = b"abc" + self.assertRaises(SystemError, _testcapi.buffer_fill_info, + source, 0, PyBUF_READ) + self.assertRaises(SystemError, _testcapi.buffer_fill_info, + source, 0, PyBUF_WRITE) + + @unittest.skipIf(_testcapi is None, "requires _testcapi") + def test_c_fill_buffer_readonly_and_writable(self): + source = b"abc" + with _testcapi.buffer_fill_info(source, 1, PyBUF_SIMPLE) as m: + self.assertEqual(bytes(m), b"abc") + self.assertTrue(m.readonly) + with _testcapi.buffer_fill_info(source, 0, PyBUF_WRITABLE) as m: + self.assertEqual(bytes(m), b"abc") + self.assertFalse(m.readonly) + self.assertRaises(BufferError, _testcapi.buffer_fill_info, + source, 1, PyBUF_WRITABLE) + + def test_inheritance(self): + class A(bytearray): + def __buffer__(self, flags): + return super().__buffer__(flags) + + a = A(b"hello") + mv = memoryview(a) + self.assertEqual(mv.tobytes(), b"hello") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_inheritance_releasebuffer(self): + rb_call_count = 0 + class B(bytearray): + def __buffer__(self, flags): + return super().__buffer__(flags) + def __release_buffer__(self, view): + nonlocal rb_call_count + rb_call_count += 1 + super().__release_buffer__(view) + + b = B(b"hello") + with memoryview(b) as mv: + self.assertEqual(mv.tobytes(), b"hello") + self.assertEqual(rb_call_count, 0) + self.assertEqual(rb_call_count, 1) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_inherit_but_return_something_else(self): + class A(bytearray): + def __buffer__(self, flags): + return memoryview(b"hello") + + a = A(b"hello") + with memoryview(a) as mv: + self.assertEqual(mv.tobytes(), b"hello") + + rb_call_count = 0 + rb_raised = False + class B(bytearray): + def __buffer__(self, flags): + return memoryview(b"hello") + def __release_buffer__(self, view): + nonlocal rb_call_count + rb_call_count += 1 + try: + super().__release_buffer__(view) + except ValueError: + nonlocal rb_raised + rb_raised = True + + b = B(b"hello") + with memoryview(b) as mv: + self.assertEqual(mv.tobytes(), b"hello") + self.assertEqual(rb_call_count, 0) + self.assertEqual(rb_call_count, 1) + self.assertIs(rb_raised, True) + + def test_override_only_release(self): + class C(bytearray): + def __release_buffer__(self, buffer): + super().__release_buffer__(buffer) + + c = C(b"hello") + with memoryview(c) as mv: + self.assertEqual(mv.tobytes(), b"hello") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_release_saves_reference(self): + smuggled_buffer = None + + class C(bytearray): + def __release_buffer__(s, buffer: memoryview): + with self.assertRaises(ValueError): + memoryview(buffer) + with self.assertRaises(ValueError): + buffer.cast("b") + with self.assertRaises(ValueError): + buffer.toreadonly() + with self.assertRaises(ValueError): + buffer[:1] + with self.assertRaises(ValueError): + buffer.__buffer__(0) + nonlocal smuggled_buffer + smuggled_buffer = buffer + self.assertEqual(buffer.tobytes(), b"hello") + super().__release_buffer__(buffer) + + c = C(b"hello") + with memoryview(c) as mv: + self.assertEqual(mv.tobytes(), b"hello") + c.clear() + with self.assertRaises(ValueError): + smuggled_buffer.tobytes() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_release_saves_reference_no_subclassing(self): + ba = bytearray(b"hello") + + class C: + def __buffer__(self, flags): + return memoryview(ba) + + def __release_buffer__(self, buffer): + self.buffer = buffer + + c = C() + with memoryview(c) as mv: + self.assertEqual(mv.tobytes(), b"hello") + self.assertEqual(c.buffer.tobytes(), b"hello") + + with self.assertRaises(BufferError): + ba.clear() + c.buffer.release() + ba.clear() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_multiple_inheritance_buffer_last(self): + class A: + def __buffer__(self, flags): + return memoryview(b"hello A") + + class B(A, bytearray): + def __buffer__(self, flags): + return super().__buffer__(flags) + + b = B(b"hello") + with memoryview(b) as mv: + self.assertEqual(mv.tobytes(), b"hello A") + + class Releaser: + def __release_buffer__(self, buffer): + self.buffer = buffer + + class C(Releaser, bytearray): + def __buffer__(self, flags): + return super().__buffer__(flags) + + c = C(b"hello C") + with memoryview(c) as mv: + self.assertEqual(mv.tobytes(), b"hello C") + c.clear() + with self.assertRaises(ValueError): + c.buffer.tobytes() + + def test_multiple_inheritance_buffer_last_raising(self): + class A: + def __buffer__(self, flags): + raise RuntimeError("should not be called") + + def __release_buffer__(self, buffer): + raise RuntimeError("should not be called") + + class B(bytearray, A): + def __buffer__(self, flags): + return super().__buffer__(flags) + + b = B(b"hello") + with memoryview(b) as mv: + self.assertEqual(mv.tobytes(), b"hello") + + class Releaser: + buffer = None + def __release_buffer__(self, buffer): + self.buffer = buffer + + class C(bytearray, Releaser): + def __buffer__(self, flags): + return super().__buffer__(flags) + + c = C(b"hello") + with memoryview(c) as mv: + self.assertEqual(mv.tobytes(), b"hello") + c.clear() + self.assertIs(c.buffer, None) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_release_buffer_with_exception_set(self): + class A: + def __buffer__(self, flags): + return memoryview(bytes(8)) + def __release_buffer__(self, view): + pass + + b = bytearray(8) + with memoryview(b): + # now b.extend will raise an exception due to exports + with self.assertRaises(BufferError): + b.extend(A()) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_bufio.py b/Lib/test/test_bufio.py index 989d8cd349b..cb9cb4d0bc7 100644 --- a/Lib/test/test_bufio.py +++ b/Lib/test/test_bufio.py @@ -28,7 +28,7 @@ def try_one(self, s): f.write(b"\n") f.write(s) f.close() - f = open(os_helper.TESTFN, "rb") + f = self.open(os_helper.TESTFN, "rb") line = f.readline() self.assertEqual(line, s + b"\n") line = f.readline() @@ -65,9 +65,6 @@ def test_nullpat(self): class CBufferSizeTest(BufferSizeTest, unittest.TestCase): open = io.open -# TODO: RUSTPYTHON -import sys -@unittest.skipIf(sys.platform == "win32", "TODO: RUSTPYTHON, can't cleanup temporary file on Windows") class PyBufferSizeTest(BufferSizeTest, unittest.TestCase): open = staticmethod(pyio.open) diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index cbba54a3bf9..cf0268c2ce5 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -1,14 +1,15 @@ # Python test set -- built-in functions import ast -import asyncio import builtins import collections +import contextlib import decimal import fractions import gc import io import locale +import math import os import pickle import platform @@ -17,6 +18,7 @@ import sys import traceback import types +import typing import unittest import warnings from contextlib import ExitStack @@ -27,10 +29,14 @@ from types import AsyncGeneratorType, FunctionType, CellType from operator import neg from test import support -from test.support import (swap_attr, maybe_get_event_loop_policy) +from test.support import cpython_only, swap_attr +from test.support import async_yield, run_yielding_async_fn +from test.support.import_helper import import_module from test.support.os_helper import (EnvironmentVarGuard, TESTFN, unlink) from test.support.script_helper import assert_python_ok +from test.support.testcase import ComplexesAreIdenticalMixin from test.support.warnings_helper import check_warnings +from test.support import requires_IEEE_754 from unittest.mock import MagicMock, patch try: import pty, signal @@ -38,6 +44,14 @@ pty = signal = None +# Detect evidence of double-rounding: sum() does not always +# get improved accuracy on machines that suffer from double rounding. +x, y = 1e16, 2.9999 # use temporary values to defeat peephole optimizer +HAVE_DOUBLE_ROUNDING = (x + y == 1e16 + 4) + +# used as proof of globals being used +A_GLOBAL_VALUE = 123 + class Squares: def __init__(self, max): @@ -134,7 +148,10 @@ def filter_char(arg): def map_char(arg): return chr(ord(arg)+1) -class BuiltinTest(unittest.TestCase): +def pack(*args): + return args + +class BuiltinTest(ComplexesAreIdenticalMixin, unittest.TestCase): # Helper to check picklability def check_iter_pickle(self, it, seq, proto): itorg = it @@ -208,6 +225,8 @@ def test_all(self): self.assertEqual(all(x > 42 for x in S), True) S = [50, 40, 60] self.assertEqual(all(x > 42 for x in S), False) + S = [50, 40, 60, TestFailingBool()] + self.assertEqual(all(x > 42 for x in S), False) def test_any(self): self.assertEqual(any([None, None, None]), False) @@ -221,9 +240,59 @@ def test_any(self): self.assertEqual(any([1, TestFailingBool()]), True) # Short-circuit S = [40, 60, 30] self.assertEqual(any(x > 42 for x in S), True) + S = [40, 60, 30, TestFailingBool()] + self.assertEqual(any(x > 42 for x in S), True) S = [10, 20, 30] self.assertEqual(any(x > 42 for x in S), False) + def test_all_any_tuple_optimization(self): + def f_all(): + return all(x-2 for x in [1,2,3]) + + def f_any(): + return any(x-1 for x in [1,2,3]) + + def f_tuple(): + return tuple(2*x for x in [1,2,3]) + + funcs = [f_all, f_any, f_tuple] + + for f in funcs: + # check that generator code object is not duplicated + code_objs = [c for c in f.__code__.co_consts if isinstance(c, type(f.__code__))] + self.assertEqual(len(code_objs), 1) + + + # check the overriding the builtins works + + global all, any, tuple + saved = all, any, tuple + try: + all = lambda x : "all" + any = lambda x : "any" + tuple = lambda x : "tuple" + + overridden_outputs = [f() for f in funcs] + finally: + all, any, tuple = saved + + self.assertEqual(overridden_outputs, ['all', 'any', 'tuple']) + + # Now repeat, overriding the builtins module as well + saved = all, any, tuple + try: + builtins.all = all = lambda x : "all" + builtins.any = any = lambda x : "any" + builtins.tuple = tuple = lambda x : "tuple" + + overridden_outputs = [f() for f in funcs] + finally: + all, any, tuple = saved + builtins.all, builtins.any, builtins.tuple = saved + + self.assertEqual(overridden_outputs, ['all', 'any', 'tuple']) + + def test_ascii(self): self.assertEqual(ascii(''), '\'\'') self.assertEqual(ascii(0), '0') @@ -299,14 +368,13 @@ class C3(C2): pass self.assertTrue(callable(c3)) def test_chr(self): + self.assertEqual(chr(0), '\0') self.assertEqual(chr(32), ' ') self.assertEqual(chr(65), 'A') self.assertEqual(chr(97), 'a') self.assertEqual(chr(0xff), '\xff') - self.assertRaises(ValueError, chr, 1<<24) - self.assertEqual(chr(sys.maxunicode), - str('\\U0010ffff'.encode("ascii"), 'unicode-escape')) self.assertRaises(TypeError, chr) + self.assertRaises(TypeError, chr, 65.0) self.assertEqual(chr(0x0000FFFF), "\U0000FFFF") self.assertEqual(chr(0x00010000), "\U00010000") self.assertEqual(chr(0x00010001), "\U00010001") @@ -318,10 +386,14 @@ def test_chr(self): self.assertEqual(chr(0x0010FFFF), "\U0010FFFF") self.assertRaises(ValueError, chr, -1) self.assertRaises(ValueError, chr, 0x00110000) - self.assertRaises((OverflowError, ValueError), chr, 2**32) + self.assertRaises(ValueError, chr, 1<<24) + self.assertRaises(ValueError, chr, 2**32-1) + self.assertRaises(ValueError, chr, -2**32) + self.assertRaises(ValueError, chr, 2**1000) + self.assertRaises(ValueError, chr, -2**1000) def test_cmp(self): - self.assertTrue(not hasattr(builtins, "cmp")) + self.assertNotHasAttr(builtins, "cmp") def test_compile(self): compile('print(1)\n', '', 'exec') @@ -360,19 +432,18 @@ def f(): """doc""" (1, False, 'doc', False, False), (2, False, None, False, False)] for optval, *expected in values: + with self.subTest(optval=optval): # test both direct compilation and compilation via AST - codeobjs = [] - codeobjs.append(compile(codestr, "", "exec", optimize=optval)) - tree = ast.parse(codestr) - codeobjs.append(compile(tree, "", "exec", optimize=optval)) - for code in codeobjs: - ns = {} - exec(code, ns) - rv = ns['f']() - self.assertEqual(rv, tuple(expected)) - - # TODO: RUSTPYTHON - @unittest.expectedFailure + codeobjs = [] + codeobjs.append(compile(codestr, "", "exec", optimize=optval)) + tree = ast.parse(codestr, optimize=optval) + codeobjs.append(compile(tree, "", "exec", optimize=optval)) + for code in codeobjs: + ns = {} + exec(code, ns) + rv = ns['f']() + self.assertEqual(rv, tuple(expected)) + def test_compile_top_level_await_no_coro(self): """Make sure top level non-await codes get the correct coroutine flags""" modes = ('single', 'exec') @@ -394,14 +465,9 @@ def test_compile_top_level_await_no_coro(self): msg=f"source={source} mode={mode}") - # TODO: RUSTPYTHON - @unittest.expectedFailure - @unittest.skipIf( - support.is_emscripten or support.is_wasi, - "socket.accept is broken" - ) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_compile_top_level_await(self): - """Test whether code some top level await can be compiled. + """Test whether code with top level await can be compiled. Make sure it compiles only with the PyCF_ALLOW_TOP_LEVEL_AWAIT flag set, and make sure the generated code object has the CO_COROUTINE flag @@ -414,12 +480,25 @@ async def arange(n): for i in range(n): yield i + class Lock: + async def __aenter__(self): + return self + + async def __aexit__(self, *exc_info): + pass + + async def sleep(delay, result=None): + assert delay == 0 + await async_yield(None) + return result + modes = ('single', 'exec') + optimizations = (-1, 0, 1, 2) code_samples = [ - '''a = await asyncio.sleep(0, result=1)''', + '''a = await sleep(0, result=1)''', '''async for i in arange(1): a = 1''', - '''async with asyncio.Lock() as l: + '''async with Lock() as l: a = 1''', '''a = [x async for x in arange(2)][1]''', '''a = 1 in {x async for x in arange(2)}''', @@ -427,45 +506,63 @@ async def arange(n): '''a = [x async for x in arange(2) async for x in arange(2)][1]''', '''a = [x async for x in (x async for x in arange(5))][1]''', '''a, = [1 for x in {x async for x in arange(1)}]''', - '''a = [await asyncio.sleep(0, x) async for x in arange(2)][1]''' + '''a = [await sleep(0, x) async for x in arange(2)][1]''', + # gh-121637: Make sure we correctly handle the case where the + # async code is optimized away + '''assert not await sleep(0); a = 1''', + '''assert [x async for x in arange(1)]; a = 1''', + '''assert {x async for x in arange(1)}; a = 1''', + '''assert {x: x async for x in arange(1)}; a = 1''', + ''' + if (a := 1) and __debug__: + async with Lock() as l: + pass + ''', + ''' + if (a := 1) and __debug__: + async for x in arange(2): + pass + ''', ] - policy = maybe_get_event_loop_policy() - try: - for mode, code_sample in product(modes, code_samples): + for mode, code_sample, optimize in product(modes, code_samples, optimizations): + with self.subTest(mode=mode, code_sample=code_sample, optimize=optimize): source = dedent(code_sample) with self.assertRaises( SyntaxError, msg=f"source={source} mode={mode}"): - compile(source, '?', mode) + compile(source, '?', mode, optimize=optimize) co = compile(source, - '?', - mode, - flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT) + '?', + mode, + flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT, + optimize=optimize) self.assertEqual(co.co_flags & CO_COROUTINE, CO_COROUTINE, - msg=f"source={source} mode={mode}") + msg=f"source={source} mode={mode}") # test we can create and advance a function type - globals_ = {'asyncio': asyncio, 'a': 0, 'arange': arange} - async_f = FunctionType(co, globals_) - asyncio.run(async_f()) + globals_ = {'Lock': Lock, 'a': 0, 'arange': arange, 'sleep': sleep} + run_yielding_async_fn(FunctionType(co, globals_)) self.assertEqual(globals_['a'], 1) # test we can await-eval, - globals_ = {'asyncio': asyncio, 'a': 0, 'arange': arange} - asyncio.run(eval(co, globals_)) + globals_ = {'Lock': Lock, 'a': 0, 'arange': arange, 'sleep': sleep} + run_yielding_async_fn(lambda: eval(co, globals_)) self.assertEqual(globals_['a'], 1) - finally: - asyncio.set_event_loop_policy(policy) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_compile_top_level_await_invalid_cases(self): # helper function just to check we can run top=level async-for async def arange(n): for i in range(n): yield i + class Lock: + async def __aenter__(self): + return self + + async def __aexit__(self, *exc_info): + pass + modes = ('single', 'exec') code_samples = [ '''def f(): await arange(10)\n''', @@ -476,30 +573,23 @@ async def arange(n): a = 1 ''', '''def f(): - async with asyncio.Lock() as l: + async with Lock() as l: a = 1 ''' ] - policy = maybe_get_event_loop_policy() - try: - for mode, code_sample in product(modes, code_samples): - source = dedent(code_sample) - with self.assertRaises( - SyntaxError, msg=f"source={source} mode={mode}"): - compile(source, '?', mode) - - with self.assertRaises( - SyntaxError, msg=f"source={source} mode={mode}"): - co = compile(source, - '?', - mode, - flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT) - finally: - asyncio.set_event_loop_policy(policy) + for mode, code_sample in product(modes, code_samples): + source = dedent(code_sample) + with self.assertRaises( + SyntaxError, msg=f"source={source} mode={mode}"): + compile(source, '?', mode) + with self.assertRaises( + SyntaxError, msg=f"source={source} mode={mode}"): + co = compile(source, + '?', + mode, + flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_compile_async_generator(self): """ With the PyCF_ALLOW_TOP_LEVEL_AWAIT flag added in 3.8, we want to @@ -509,13 +599,35 @@ def test_compile_async_generator(self): code = dedent("""async def ticker(): for i in range(10): yield i - await asyncio.sleep(0)""") + await sleep(0)""") co = compile(code, '?', 'exec', flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT) glob = {} exec(co, glob) self.assertEqual(type(glob['ticker']()), AsyncGeneratorType) + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: <_ast.Name object at 0xb40000731e3d1360> is not an instance of + def test_compile_ast(self): + args = ("a*__debug__", "f.py", "exec") + raw = compile(*args, flags = ast.PyCF_ONLY_AST).body[0] + opt1 = compile(*args, flags = ast.PyCF_OPTIMIZED_AST).body[0] + opt2 = compile(ast.parse(args[0]), *args[1:], flags = ast.PyCF_OPTIMIZED_AST).body[0] + + for tree in (raw, opt1, opt2): + self.assertIsInstance(tree.value, ast.BinOp) + self.assertIsInstance(tree.value.op, ast.Mult) + self.assertIsInstance(tree.value.left, ast.Name) + self.assertEqual(tree.value.left.id, 'a') + + raw_right = raw.value.right + self.assertIsInstance(raw_right, ast.Name) + self.assertEqual(raw_right.id, "__debug__") + + for opt in [opt1, opt2]: + opt_right = opt.value.right + self.assertIsInstance(opt_right, ast.Constant) + self.assertEqual(opt_right.value, __debug__) + def test_delattr(self): sys.spam = 1 delattr(sys, 'spam') @@ -524,8 +636,7 @@ def test_delattr(self): msg = r"^attribute name must be string, not 'int'$" self.assertRaisesRegex(TypeError, msg, delattr, sys, 1) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: '__repr__' unexpectedly found in ['__class__', '__delattr__', '__dict__', '__dir__', '__doc__', '__eq__', '__firstlineno__', '__format__', '__ge__', '__getattribute__', '__getstate__', '__gt__', '__hash__', '__init__', '__init_subclass__', '__le__', '__lt__', '__module__', '__ne__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__sizeof__', '__slots__', '__str__', '__subclasshook__', 'bar'] def test_dir(self): # dir(wrong number of arguments) self.assertRaises(TypeError, dir, 42, 42) @@ -587,6 +698,14 @@ def __dir__(self): self.assertIsInstance(res, list) self.assertTrue(res == ["a", "b", "c"]) + # dir(obj__dir__iterable) + class Foo(object): + def __dir__(self): + return {"b", "c", "a"} + res = dir(Foo()) + self.assertIsInstance(res, list) + self.assertEqual(sorted(res), ["a", "b", "c"]) + # dir(obj__dir__not_sequence) class Foo(object): def __dir__(self): @@ -625,6 +744,16 @@ def test_divmod(self): self.assertAlmostEqual(result[1], exp_result[1]) self.assertRaises(TypeError, divmod) + self.assertRaisesRegex( + ZeroDivisionError, + "division by zero", + divmod, 1, 0, + ) + self.assertRaisesRegex( + ZeroDivisionError, + "division by zero", + divmod, 0.0, 0, + ) def test_eval(self): self.assertEqual(eval('1+1'), 2) @@ -649,6 +778,11 @@ def __getitem__(self, key): raise ValueError self.assertRaises(ValueError, eval, "foo", {}, X()) + def test_eval_kwargs(self): + data = {"A_GLOBAL_VALUE": 456} + self.assertEqual(eval("globals()['A_GLOBAL_VALUE']", globals=data), 456) + self.assertEqual(eval("globals()['A_GLOBAL_VALUE']", locals=data), 123) + def test_general_eval(self): # Tests that general mappings can be used for the locals argument @@ -742,8 +876,19 @@ def test_exec(self): del l['__builtins__'] self.assertEqual((g, l), ({'a': 1}, {'b': 2})) - # TODO: RUSTPYTHON - @unittest.expectedFailure + def test_exec_kwargs(self): + g = {} + exec('global z\nz = 1', globals=g) + if '__builtins__' in g: + del g['__builtins__'] + self.assertEqual(g, {'z': 1}) + + # if we only set locals, the global assignment will not + # reach this locals dictionary + g = {} + exec('global z\nz = 1', locals=g) + self.assertEqual(g, {}) + def test_exec_globals(self): code = compile("print('Hello World!')", "", "exec") # no builtin function @@ -753,8 +898,6 @@ def test_exec_globals(self): self.assertRaises(TypeError, exec, code, {'__builtins__': 123}) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_exec_globals_frozen(self): class frozendict_error(Exception): pass @@ -787,8 +930,6 @@ def __setitem__(self, key, value): self.assertRaises(frozendict_error, exec, code, namespace) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_exec_globals_error_on_get(self): # custom `globals` or `builtins` can raise errors on item access class setonlyerror(Exception): @@ -808,8 +949,6 @@ def __getitem__(self, key): self.assertRaises(setonlyerror, exec, code, {'__builtins__': setonlydict({'superglobal': 1})}) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_exec_globals_dict_subclass(self): class customdict(dict): # this one should not do anything fancy pass @@ -821,6 +960,34 @@ class customdict(dict): # this one should not do anything fancy self.assertRaisesRegex(NameError, "name 'superglobal' is not defined", exec, code, {'__builtins__': customdict()}) + def test_eval_builtins_mapping(self): + code = compile("superglobal", "test", "eval") + # works correctly + ns = {'__builtins__': types.MappingProxyType({'superglobal': 1})} + self.assertEqual(eval(code, ns), 1) + # custom builtins mapping is missing key + ns = {'__builtins__': types.MappingProxyType({})} + self.assertRaisesRegex(NameError, "name 'superglobal' is not defined", + eval, code, ns) + + @unittest.expectedFailure # TODO: RUSTPYTHON; wrong error message + def test_exec_builtins_mapping_import(self): + code = compile("import foo.bar", "test", "exec") + ns = {'__builtins__': types.MappingProxyType({})} + self.assertRaisesRegex(ImportError, "__import__ not found", exec, code, ns) + ns = {'__builtins__': types.MappingProxyType({'__import__': lambda *args: args})} + exec(code, ns) + self.assertEqual(ns['foo'], ('foo.bar', ns, ns, None, 0)) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: AttributeError not raised by eval + def test_eval_builtins_mapping_reduce(self): + # list_iterator.__reduce__() calls _PyEval_GetBuiltin("iter") + code = compile("x.__reduce__()", "test", "eval") + ns = {'__builtins__': types.MappingProxyType({}), 'x': iter([1, 2])} + self.assertRaisesRegex(AttributeError, "iter", eval, code, ns) + ns = {'__builtins__': types.MappingProxyType({'iter': iter}), 'x': iter([1, 2])} + self.assertEqual(eval(code, ns), (iter, ([1, 2],), 0)) + def test_exec_redirected(self): savestdout = sys.stdout sys.stdout = None # Whatever that cannot flush() @@ -832,8 +999,7 @@ def test_exec_redirected(self): finally: sys.stdout = savestdout - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Unexpected keyword argument closure def test_exec_closure(self): def function_without_closures(): return 3 * 5 @@ -901,8 +1067,24 @@ def four_freevars(): three_freevars.__code__, three_freevars.__globals__, closure=my_closure) + my_closure = tuple(my_closure) + + # should fail: anything passed to closure= isn't allowed + # when the source is a string + self.assertRaises(TypeError, + exec, + "pass", + closure=int) + + # should fail: correct closure= argument isn't allowed + # when the source is a string + self.assertRaises(TypeError, + exec, + "pass", + closure=my_closure) # should fail: closure tuple with one non-cell-var + my_closure = list(my_closure) my_closure[0] = int my_closure = tuple(my_closure) self.assertRaises(TypeError, @@ -943,6 +1125,20 @@ def test_filter_pickle(self): f2 = filter(filter_char, "abcdeabcde") self.check_iter_pickle(f1, list(f2), proto) + @unittest.skip("TODO: RUSTPYTHON; Segfault") + @support.skip_wasi_stack_overflow() + @support.skip_emscripten_stack_overflow() + @support.requires_resource('cpu') + def test_filter_dealloc(self): + # Tests recursive deallocation of nested filter objects using the + # thrashcan mechanism. See gh-102356 for more details. + max_iters = 1000000 + i = filter(bool, range(max_iters)) + for _ in range(max_iters): + i = filter(bool, i) + del i + gc.collect() + def test_getattr(self): self.assertTrue(getattr(sys, 'stdout') is sys.stdout) self.assertRaises(TypeError, getattr) @@ -994,6 +1190,16 @@ def __hash__(self): return self self.assertEqual(hash(Z(42)), hash(42)) + def test_invalid_hash_typeerror(self): + # GH-140406: The returned object from __hash__() would leak if it + # wasn't an integer. + class A: + def __hash__(self): + return 1.0 + + with self.assertRaises(TypeError): + hash(A()) + def test_hex(self): self.assertEqual(hex(16), '0x10') self.assertEqual(hex(-16), '-0x10') @@ -1155,6 +1361,124 @@ def test_map_pickle(self): m2 = map(map_char, "Is this the real life?") self.check_iter_pickle(m1, list(m2), proto) + # strict map tests based on strict zip tests + + def test_map_pickle_strict(self): + a = (1, 2, 3) + b = (4, 5, 6) + t = [(1, 4), (2, 5), (3, 6)] + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + m1 = map(pack, a, b, strict=True) + self.check_iter_pickle(m1, t, proto) + + def test_map_pickle_strict_fail(self): + a = (1, 2, 3) + b = (4, 5, 6, 7) + t = [(1, 4), (2, 5), (3, 6)] + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + m1 = map(pack, a, b, strict=True) + m2 = pickle.loads(pickle.dumps(m1, proto)) + self.assertEqual(self.iter_error(m1, ValueError), t) + self.assertEqual(self.iter_error(m2, ValueError), t) + + def test_map_strict(self): + self.assertEqual(tuple(map(pack, (1, 2, 3), 'abc', strict=True)), + ((1, 'a'), (2, 'b'), (3, 'c'))) + self.assertRaises(ValueError, tuple, + map(pack, (1, 2, 3, 4), 'abc', strict=True)) + self.assertRaises(ValueError, tuple, + map(pack, (1, 2), 'abc', strict=True)) + self.assertRaises(ValueError, tuple, + map(pack, (1, 2), (1, 2), 'abc', strict=True)) + + # gh-140517: Testing refleaks with mortal objects. + t1 = (None, object()) + t2 = (object(), object()) + t3 = (object(),) + + self.assertRaises(ValueError, tuple, + map(pack, t1, 'a', strict=True)) + self.assertRaises(ValueError, tuple, + map(pack, t1, t2, 'a', strict=True)) + self.assertRaises(ValueError, tuple, + map(pack, t1, t2, t3, strict=True)) + self.assertRaises(ValueError, tuple, + map(pack, 'a', t1, strict=True)) + self.assertRaises(ValueError, tuple, + map(pack, 'a', t2, t3, strict=True)) + + def test_map_strict_iterators(self): + x = iter(range(5)) + y = [0] + z = iter(range(5)) + self.assertRaises(ValueError, list, + (map(pack, x, y, z, strict=True))) + self.assertEqual(next(x), 2) + self.assertEqual(next(z), 1) + + def test_map_strict_error_handling(self): + + class Error(Exception): + pass + + class Iter: + def __init__(self, size): + self.size = size + def __iter__(self): + return self + def __next__(self): + self.size -= 1 + if self.size < 0: + raise Error + return self.size + + l1 = self.iter_error(map(pack, "AB", Iter(1), strict=True), Error) + self.assertEqual(l1, [("A", 0)]) + l2 = self.iter_error(map(pack, "AB", Iter(2), "A", strict=True), ValueError) + self.assertEqual(l2, [("A", 1, "A")]) + l3 = self.iter_error(map(pack, "AB", Iter(2), "ABC", strict=True), Error) + self.assertEqual(l3, [("A", 1, "A"), ("B", 0, "B")]) + l4 = self.iter_error(map(pack, "AB", Iter(3), strict=True), ValueError) + self.assertEqual(l4, [("A", 2), ("B", 1)]) + l5 = self.iter_error(map(pack, Iter(1), "AB", strict=True), Error) + self.assertEqual(l5, [(0, "A")]) + l6 = self.iter_error(map(pack, Iter(2), "A", strict=True), ValueError) + self.assertEqual(l6, [(1, "A")]) + l7 = self.iter_error(map(pack, Iter(2), "ABC", strict=True), Error) + self.assertEqual(l7, [(1, "A"), (0, "B")]) + l8 = self.iter_error(map(pack, Iter(3), "AB", strict=True), ValueError) + self.assertEqual(l8, [(2, "A"), (1, "B")]) + + def test_map_strict_error_handling_stopiteration(self): + + class Iter: + def __init__(self, size): + self.size = size + def __iter__(self): + return self + def __next__(self): + self.size -= 1 + if self.size < 0: + raise StopIteration + return self.size + + l1 = self.iter_error(map(pack, "AB", Iter(1), strict=True), ValueError) + self.assertEqual(l1, [("A", 0)]) + l2 = self.iter_error(map(pack, "AB", Iter(2), "A", strict=True), ValueError) + self.assertEqual(l2, [("A", 1, "A")]) + l3 = self.iter_error(map(pack, "AB", Iter(2), "ABC", strict=True), ValueError) + self.assertEqual(l3, [("A", 1, "A"), ("B", 0, "B")]) + l4 = self.iter_error(map(pack, "AB", Iter(3), strict=True), ValueError) + self.assertEqual(l4, [("A", 2), ("B", 1)]) + l5 = self.iter_error(map(pack, Iter(1), "AB", strict=True), ValueError) + self.assertEqual(l5, [(0, "A")]) + l6 = self.iter_error(map(pack, Iter(2), "A", strict=True), ValueError) + self.assertEqual(l6, [(1, "A")]) + l7 = self.iter_error(map(pack, Iter(2), "ABC", strict=True), ValueError) + self.assertEqual(l7, [(1, "A"), (0, "B")]) + l8 = self.iter_error(map(pack, Iter(3), "AB", strict=True), ValueError) + self.assertEqual(l8, [(2, "A"), (1, "B")]) + def test_max(self): self.assertEqual(max('123123'), '3') self.assertEqual(max(1, 2, 3), 3) @@ -1172,7 +1496,11 @@ def test_max(self): max() self.assertRaises(TypeError, max, 42) - self.assertRaises(ValueError, max, ()) + with self.assertRaisesRegex( + ValueError, + r'max\(\) iterable argument is empty' + ): + max(()) class BadSeq: def __getitem__(self, index): raise ValueError @@ -1231,7 +1559,11 @@ def test_min(self): min() self.assertRaises(TypeError, min, 42) - self.assertRaises(ValueError, min, ()) + with self.assertRaisesRegex( + ValueError, + r'min\(\) iterable argument is empty' + ): + min(()) class BadSeq: def __getitem__(self, index): raise ValueError @@ -1332,18 +1664,13 @@ def test_open(self): self.assertRaises(ValueError, open, 'a\x00b') self.assertRaises(ValueError, open, b'a\x00b') - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf(sys.flags.utf8_mode, "utf-8 mode is enabled") def test_open_default_encoding(self): - old_environ = dict(os.environ) - try: + with EnvironmentVarGuard() as env: # try to get a user preferred encoding different than the current # locale encoding to check that open() uses the current locale # encoding and not the user preferred encoding - for key in ('LC_ALL', 'LANG', 'LC_CTYPE'): - if key in os.environ: - del os.environ[key] + env.unset('LC_ALL', 'LANG', 'LC_CTYPE') self.write_testfile() current_locale_encoding = locale.getencoding() @@ -1352,9 +1679,6 @@ def test_open_default_encoding(self): fp = open(TESTFN, 'w') with fp: self.assertEqual(fp.encoding, current_locale_encoding) - finally: - os.environ.clear() - os.environ.update(old_environ) @support.requires_subprocess() def test_open_non_inheritable(self): @@ -1486,6 +1810,29 @@ def test_input(self): sys.stdout = savestdout fp.close() + def test_input_gh130163(self): + class X(io.StringIO): + def __getattribute__(self, name): + nonlocal patch + if patch: + patch = False + sys.stdout = X() + sys.stderr = X() + sys.stdin = X('input\n') + support.gc_collect() + return io.StringIO.__getattribute__(self, name) + + with (support.swap_attr(sys, 'stdout', None), + support.swap_attr(sys, 'stderr', None), + support.swap_attr(sys, 'stdin', None)): + patch = False + # the only references: + sys.stdout = X() + sys.stderr = X() + sys.stdin = X('input\n') + patch = True + input() # should not crash + # test_int(): see test_int.py for tests of built-in function int(). def test_repr(self): @@ -1501,6 +1848,11 @@ def test_repr(self): a[0] = a self.assertEqual(repr(a), '{0: {...}}') + def test_repr_blocked(self): + class C: + __repr__ = None + self.assertRaises(TypeError, repr, C()) + def test_round(self): self.assertEqual(round(0.0), 0.0) self.assertEqual(type(round(0.0)), int) @@ -1607,14 +1959,17 @@ def test_bug_27936(self): def test_setattr(self): setattr(sys, 'spam', 1) - self.assertEqual(sys.spam, 1) + try: + self.assertEqual(sys.spam, 1) + finally: + del sys.spam self.assertRaises(TypeError, setattr) self.assertRaises(TypeError, setattr, sys) self.assertRaises(TypeError, setattr, sys, 'spam') msg = r"^attribute name must be string, not 'int'$" self.assertRaisesRegex(TypeError, msg, setattr, sys, 1, 'spam') - # test_str(): see test_unicode.py and test_bytes.py for str() tests. + # test_str(): see test_str.py and test_bytes.py for str() tests. def test_sum(self): self.assertEqual(sum([]), 0) @@ -1644,6 +1999,8 @@ def test_sum(self): self.assertEqual(repr(sum([-0.0])), '0.0') self.assertEqual(repr(sum([-0.0], -0.0)), '-0.0') self.assertEqual(repr(sum([], -0.0)), '-0.0') + self.assertTrue(math.isinf(sum([float("inf"), float("inf")]))) + self.assertTrue(math.isinf(sum([1e308, 1e308]))) self.assertRaises(TypeError, sum) self.assertRaises(TypeError, sum, 42) @@ -1658,6 +2015,8 @@ def test_sum(self): self.assertRaises(TypeError, sum, [], '') self.assertRaises(TypeError, sum, [], b'') self.assertRaises(TypeError, sum, [], bytearray()) + self.assertRaises(OverflowError, sum, [1.0, 10**1000]) + self.assertRaises(OverflowError, sum, [1j, 10**1000]) class BadSeq: def __getitem__(self, index): @@ -1668,6 +2027,37 @@ def __getitem__(self, index): sum(([x] for x in range(10)), empty) self.assertEqual(empty, []) + xs = [complex(random.random() - .5, random.random() - .5) + for _ in range(10000)] + self.assertEqual(sum(xs), complex(sum(z.real for z in xs), + sum(z.imag for z in xs))) + + # test that sum() of complex and real numbers doesn't + # smash sign of imaginary 0 + self.assertComplexesAreIdentical(sum([complex(1, -0.0), 1]), + complex(2, -0.0)) + self.assertComplexesAreIdentical(sum([1, complex(1, -0.0)]), + complex(2, -0.0)) + self.assertComplexesAreIdentical(sum([complex(1, -0.0), 1.0]), + complex(2, -0.0)) + self.assertComplexesAreIdentical(sum([1.0, complex(1, -0.0)]), + complex(2, -0.0)) + + @requires_IEEE_754 + @unittest.skipIf(HAVE_DOUBLE_ROUNDING, + "sum accuracy not guaranteed on machines with double rounding") + @support.cpython_only # Other implementations may choose a different algorithm + def test_sum_accuracy(self): + self.assertEqual(sum([0.1] * 10), 1.0) + self.assertEqual(sum([1.0, 10E100, 1.0, -10E100]), 2.0) + self.assertEqual(sum([1.0, 10E100, 1.0, -10E100, 2j]), 2+2j) + self.assertEqual(sum([2+1j, 10E100j, 1j, -10E100j]), 2+2j) + self.assertEqual(sum([1j, 1, 10E100j, 1j, 1.0, -10E100j]), 2+2j) + self.assertEqual(sum([2j, 1., 10E100, 1., -10E100]), 2+2j) + self.assertEqual(sum([1.0, 10**100, 1.0, -10**100]), 2.0) + self.assertEqual(sum([2j, 1.0, 10**100, 1.0, -10**100]), 2+2j) + self.assertEqual(sum([0.1j]*10 + [fractions.Fraction(1, 10)]), 0.1+1j) + def test_type(self): self.assertEqual(type(''), type('123')) self.assertNotEqual(type(''), type(())) @@ -1947,7 +2337,7 @@ def __format__(self, format_spec): # tests for object.__format__ really belong elsewhere, but # there's no good place to put them x = object().__format__('') - self.assertTrue(x.startswith(' eval() roundtrip - if stdio_encoding: - expected = terminal_input.decode(stdio_encoding, 'surrogateescape') - else: - expected = terminal_input.decode(sys.stdin.encoding) # what else? + if expected is None: + if stdio_encoding: + expected = terminal_input.decode(stdio_encoding, 'surrogateescape') + else: + expected = terminal_input.decode(sys.stdin.encoding) # what else? self.assertEqual(input_result, expected) - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_input_tty(self): - # Test input() functionality when wired to a tty (the code path - # is different and invokes GNU readline if available). - self.check_input_tty("prompt", b"quux") - - def skip_if_readline(self): + @contextlib.contextmanager + def detach_readline(self): # bpo-13886: When the readline module is loaded, PyOS_Readline() uses # the readline implementation. In some cases, the Python readline # callback rlhandler() is called by readline with a string without - # non-ASCII characters. Skip tests on non-ASCII characters if the - # readline module is loaded, since test_builtin is not intented to test + # non-ASCII characters. + # Unlink readline temporarily from PyOS_Readline() for those tests, + # since test_builtin is not intended to test # the readline module, but the builtins module. - if 'readline' in sys.modules: - self.skipTest("the readline module is loaded") + if "readline" in sys.modules: + c = import_module("ctypes") + fp_api = "PyOS_ReadlineFunctionPointer" + prev_value = c.c_void_p.in_dll(c.pythonapi, fp_api).value + c.c_void_p.in_dll(c.pythonapi, fp_api).value = None + try: + yield + finally: + c.c_void_p.in_dll(c.pythonapi, fp_api).value = prev_value + else: + yield + + def test_input_tty(self): + # Test input() functionality when wired to a tty + self.check_input_tty("prompt", b"quux") - @unittest.skipUnless(hasattr(sys.stdin, 'detach'), 'TODO: RustPython: requires detach function in TextIOWrapper') - @unittest.expectedFailure # TODO: RUSTPYTHON AssertionError: got 0 lines in pipe but expected 2, child output was: quux + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: got 0 lines in pipe but expected 2, child output was: quux def test_input_tty_non_ascii(self): - self.skip_if_readline() # Check stdin/stdout encoding is used when invoking PyOS_Readline() - self.check_input_tty("prompté", b"quux\xe9", "utf-8") + self.check_input_tty("prompté", b"quux\xc3\xa9", "utf-8") - @unittest.skipUnless(hasattr(sys.stdin, 'detach'), 'TODO: RustPython: requires detach function in TextIOWrapper') - @unittest.expectedFailure # TODO: RUSTPYTHON AssertionError: got 0 lines in pipe but expected 2, child output was: quux + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: got 0 lines in pipe but expected 2, child output was: quux def test_input_tty_non_ascii_unicode_errors(self): - self.skip_if_readline() # Check stdin/stdout error handler is used when invoking PyOS_Readline() self.check_input_tty("prompté", b"quux\xe9", "ascii") - @unittest.skip('TODO: RUSTPYTHON FAILURE, WORKER BUG') - @unittest.expectedFailure # TODO: RUSTPYTHON AssertionError: got 0 lines in pipe but expected 2, child output was: quux + def test_input_tty_null_in_prompt(self): + self.check_input_tty("prompt\0", b"", + expected='ValueError: input: prompt string cannot contain ' + 'null characters') + + def test_input_tty_nonencodable_prompt(self): + self.check_input_tty("prompté", b"quux", "ascii", stdout_errors='strict', + expected="UnicodeEncodeError: 'ascii' codec can't encode " + "character '\\xe9' in position 6: ordinal not in " + "range(128)") + + def test_input_tty_nondecodable_input(self): + self.check_input_tty("prompt", b"quux\xe9", "ascii", stdin_errors='strict', + expected="UnicodeDecodeError: 'ascii' codec can't decode " + "byte 0xe9 in position 4: ordinal not in " + "range(128)") + + @unittest.skip("TODO: RUSTPYTHON; FAILURE, WORKER BUG") + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: got 0 lines in pipe but expected 2, child output was: quux def test_input_no_stdout_fileno(self): # Issue #24402: If stdin is the original terminal but stdout.fileno() # fails, do not use the original stdout file descriptor @@ -2330,8 +2789,6 @@ def test_baddecorator(self): class ShutdownTest(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_cleanup(self): # Issue #19255: builtins are still available at shutdown code = """if 1: @@ -2364,6 +2821,35 @@ def __del__(self): self.assertEqual(["before", "after"], out.decode().splitlines()) +@cpython_only +class ImmortalTests(unittest.TestCase): + + if sys.maxsize < (1 << 32): + IMMORTAL_REFCOUNT_MINIMUM = 1 << 30 + else: + IMMORTAL_REFCOUNT_MINIMUM = 1 << 31 + + IMMORTALS = (None, True, False, Ellipsis, NotImplemented, *range(-5, 257)) + + def assert_immortal(self, immortal): + with self.subTest(immortal): + self.assertGreater(sys.getrefcount(immortal), self.IMMORTAL_REFCOUNT_MINIMUM) + + def test_immortals(self): + for immortal in self.IMMORTALS: + self.assert_immortal(immortal) + + def test_list_repeat_respect_immortality(self): + refs = list(self.IMMORTALS) * 42 + for immortal in self.IMMORTALS: + self.assert_immortal(immortal) + + def test_tuple_repeat_respect_immortality(self): + refs = tuple(self.IMMORTALS) * 42 + for immortal in self.IMMORTALS: + self.assert_immortal(immortal) + + class TestType(unittest.TestCase): def test_new_type(self): A = type('A', (), {}) @@ -2372,6 +2858,7 @@ def test_new_type(self): self.assertEqual(A.__module__, __name__) self.assertEqual(A.__bases__, (object,)) self.assertIs(A.__base__, object) + self.assertNotIn('__firstlineno__', A.__dict__) x = A() self.assertIs(type(x), A) self.assertIs(x.__class__, A) @@ -2450,6 +2937,29 @@ def test_type_qualname(self): A.__qualname__ = b'B' self.assertEqual(A.__qualname__, 'D.E') + def test_type_firstlineno(self): + A = type('A', (), {'__firstlineno__': 42}) + self.assertEqual(A.__name__, 'A') + self.assertEqual(A.__module__, __name__) + self.assertEqual(A.__dict__['__firstlineno__'], 42) + A.__module__ = 'testmodule' + self.assertEqual(A.__module__, 'testmodule') + self.assertNotIn('__firstlineno__', A.__dict__) + A.__firstlineno__ = 43 + self.assertEqual(A.__dict__['__firstlineno__'], 43) + + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Expected type 'tuple' but 'str' found. + def test_type_typeparams(self): + class A[T]: + pass + T, = A.__type_params__ + self.assertIsInstance(T, typing.TypeVar) + A.__type_params__ = "whatever" + self.assertEqual(A.__type_params__, "whatever") + with self.assertRaises(TypeError): + del A.__type_params__ + self.assertEqual(A.__type_params__, "whatever") + def test_type_doc(self): for doc in 'x', '\xc4', '\U0001f40d', 'x\x00y', b'x', 42, None: A = type('A', (), {'__doc__': doc}) @@ -2521,7 +3031,8 @@ def test_namespace_order(self): def load_tests(loader, tests, pattern): from doctest import DocTestSuite - tests.addTest(DocTestSuite(builtins)) + if sys.float_repr_style == 'short': + tests.addTest(DocTestSuite(builtins)) return tests if __name__ == "__main__": diff --git a/Lib/test/test_bz2.py b/Lib/test/test_bz2.py index 26b5e79d337..148d8f98c79 100644 --- a/Lib/test/test_bz2.py +++ b/Lib/test/test_bz2.py @@ -730,7 +730,7 @@ def testOpenBytesFilename(self): self.assertEqual(f.read(), self.DATA) self.assertEqual(f.name, str_filename) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: != 'Z:\\TEMP\\tmphoipjcen' def testOpenPathLikeFilename(self): filename = FakePath(self.filename) with BZ2File(filename, "wb") as f: @@ -1189,7 +1189,6 @@ def test_encoding_error_handler(self): as f: self.assertEqual(f.read(), "foobar") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_newline(self): # Test with explicit newline (universal newline mode disabled). text = self.TEXT.decode("ascii") diff --git a/Lib/test/test_c_locale_coercion.py b/Lib/test/test_c_locale_coercion.py index 818dc16b834..71f934756e2 100644 --- a/Lib/test/test_c_locale_coercion.py +++ b/Lib/test/test_c_locale_coercion.py @@ -243,8 +243,6 @@ def setUpClass(cls): if not AVAILABLE_TARGETS: raise unittest.SkipTest("No C-with-UTF-8 locale available") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_external_target_locale_configuration(self): # Explicitly setting a target locale should give the same behaviour as diff --git a/Lib/test/test_class.py b/Lib/test/test_class.py index a70a8078977..7420f289b16 100644 --- a/Lib/test/test_class.py +++ b/Lib/test/test_class.py @@ -1,6 +1,7 @@ "Test the functionality of Python classes implementing operators." import unittest +from test import support from test.support import cpython_only, import_helper, script_helper testmeths = [ @@ -134,6 +135,7 @@ def __%s__(self, *args): AllTests = type("AllTests", (object,), d) del d, statictests, method, method_template +@support.thread_unsafe("callLst is shared between threads") class ClassTests(unittest.TestCase): def setUp(self): callLst[:] = [] @@ -554,7 +556,9 @@ class Custom: self.assertFalse(hasattr(o, "__call__")) self.assertFalse(hasattr(c, "__call__")) - @unittest.skip("TODO: RUSTPYTHON, segmentation fault") + @unittest.skip("TODO: RUSTPYTHON; segmentation fault") + @support.skip_emscripten_stack_overflow() + @support.skip_wasi_stack_overflow() def testSFBug532646(self): # Test for SF bug 532646 @@ -570,8 +574,7 @@ class A: else: self.fail("Failed to raise RecursionError") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def testForExceptionsRaisedInInstanceGetattr2(self): # Tests for exceptions raised in instance_getattr2(). @@ -688,8 +691,7 @@ class A: with self.assertRaisesRegex(AttributeError, error_msg): del A.x - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def testObjectAttributeAccessErrorMessages(self): class A: pass @@ -739,8 +741,6 @@ def __setattr__(self, name, value) -> None: with self.assertRaisesRegex(AttributeError, error_msg): del B().z - # TODO: RUSTPYTHON - @unittest.expectedFailure def testConstructorErrorMessages(self): # bpo-31506: Improves the error message logic for object_new & object_init @@ -845,12 +845,28 @@ def __init__(self, obj): Type(i) self.assertEqual(calls, 100) -try: - from _testinternalcapi import has_inline_values -except ImportError: - has_inline_values = None + def test_specialization_class_call_doesnt_crash(self): + # gh-123185 + + class Foo: + def __init__(self, arg): + pass + + for _ in range(8): + try: + Foo() + except: + pass + + +# from _testinternalcapi import has_inline_values # XXX: RUSTPYTHON + +Py_TPFLAGS_INLINE_VALUES = (1 << 2) +Py_TPFLAGS_MANAGED_DICT = (1 << 4) + +class NoManagedDict: + __slots__ = ('a',) -Py_TPFLAGS_MANAGED_DICT = (1 << 2) class Plain: pass @@ -865,38 +881,55 @@ def __init__(self): self.d = 4 +class VarSizedSubclass(tuple): + pass + + class TestInlineValues(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_flags(self): - self.assertEqual(Plain.__flags__ & Py_TPFLAGS_MANAGED_DICT, Py_TPFLAGS_MANAGED_DICT) - self.assertEqual(WithAttrs.__flags__ & Py_TPFLAGS_MANAGED_DICT, Py_TPFLAGS_MANAGED_DICT) + @unittest.expectedFailure # TODO: RUSTPYTHON; NameError: name 'has_inline_values' is not defined. + def test_no_flags_for_slots_class(self): + flags = NoManagedDict.__flags__ + self.assertEqual(flags & Py_TPFLAGS_MANAGED_DICT, 0) + self.assertEqual(flags & Py_TPFLAGS_INLINE_VALUES, 0) + self.assertFalse(has_inline_values(NoManagedDict())) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 0 != 4 + def test_both_flags_for_regular_class(self): + for cls in (Plain, WithAttrs): + with self.subTest(cls=cls.__name__): + flags = cls.__flags__ + self.assertEqual(flags & Py_TPFLAGS_MANAGED_DICT, Py_TPFLAGS_MANAGED_DICT) + self.assertEqual(flags & Py_TPFLAGS_INLINE_VALUES, Py_TPFLAGS_INLINE_VALUES) + self.assertTrue(has_inline_values(cls())) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 0 != 4 + def test_managed_dict_only_for_varsized_subclass(self): + flags = VarSizedSubclass.__flags__ + self.assertEqual(flags & Py_TPFLAGS_MANAGED_DICT, Py_TPFLAGS_MANAGED_DICT) + self.assertEqual(flags & Py_TPFLAGS_INLINE_VALUES, 0) + self.assertFalse(has_inline_values(VarSizedSubclass())) + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_has_inline_values(self): c = Plain() self.assertTrue(has_inline_values(c)) del c.__dict__ self.assertFalse(has_inline_values(c)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_instances(self): self.assertTrue(has_inline_values(Plain())) self.assertTrue(has_inline_values(WithAttrs())) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_inspect_dict(self): for cls in (Plain, WithAttrs): c = cls() c.__dict__ self.assertTrue(has_inline_values(c)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_update_dict(self): d = { "e": 5, "f": 6 } for cls in (Plain, WithAttrs): @@ -913,8 +946,7 @@ def check_100(self, obj): for i in range(100): self.assertEqual(getattr(obj, f"a{i}"), i) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_many_attributes(self): class C: pass c = C() @@ -925,8 +957,7 @@ class C: pass c = C() self.assertTrue(has_inline_values(c)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_many_attributes_with_dict(self): class C: pass c = C() @@ -947,8 +978,7 @@ def __init__(self): obj.foo = None # Aborted here self.assertEqual(obj.__dict__, {"foo":None}) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_store_attr_deleted_dict(self): class Foo: pass @@ -958,8 +988,7 @@ class Foo: f.a = 3 self.assertEqual(f.a, 3) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_rematerialize_object_dict(self): # gh-121860: rematerializing an object's managed dictionary after it # had been deleted caused a crash. @@ -978,7 +1007,7 @@ class Bar: pass self.assertIsInstance(f, Bar) self.assertEqual(f.__dict__, {}) - @unittest.skip("TODO: RUSTPYTHON, unexpectedly long runtime") + @unittest.skip("TODO: RUSTPYTHON; unexpectedly long runtime") def test_store_attr_type_cache(self): """Verifies that the type cache doesn't provide a value which is inconsistent from the dict.""" @@ -1029,5 +1058,6 @@ def __init__(self): self.assertFalse(out, msg=out.decode('utf-8')) self.assertFalse(err, msg=err.decode('utf-8')) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index eb455ebaed7..8aa6dad2735 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -475,8 +475,6 @@ def test_unmached_quote(self): self.assertRegex(err.decode('ascii', 'ignore'), 'SyntaxError') self.assertEqual(b'', out) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_stdout_flush_at_shutdown(self): # Issue #5319: if stdout.flush() fails at shutdown, an error should # be printed out. diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index d2b3a7d3e40..f977b97bcd3 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -548,8 +548,6 @@ def test_dash_m_main_traceback(self): self.assertIn(b'Exception in __main__ module', err) self.assertIn(b'Traceback', err) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_pep_409_verbiage(self): # Make sure PEP 409 syntax properly suppresses # the context of an exception @@ -620,7 +618,6 @@ def test_syntaxerror_unindented_caret_position(self): # Confirm that the caret is located under the '=' sign self.assertIn("\n ^^^^^\n", text) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_syntaxerror_indented_caret_position(self): script = textwrap.dedent("""\ if True: diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index f2ef233a59a..b4b15e29f26 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -6,8 +6,7 @@ ... return g ... -# TODO: RUSTPYTHON ->>> # dump(f.__code__) +>>> dump(f.__code__) name: f argcount: 1 posonlyargcount: 0 @@ -18,10 +17,9 @@ freevars: () nlocals: 2 flags: 3 -consts: ('None', '') +consts: ('',) -# TODO: RUSTPYTHON ->>> # dump(f(4).__code__) +>>> dump(f(4).__code__) name: g argcount: 1 posonlyargcount: 0 @@ -41,8 +39,7 @@ ... return c ... -# TODO: RUSTPYTHON ->>> # dump(h.__code__) +>>> dump(h.__code__) name: h argcount: 2 posonlyargcount: 0 @@ -60,8 +57,7 @@ ... print(obj.attr2) ... print(obj.attr3) -# TODO: RUSTPYTHON ->>> # dump(attrs.__code__) +>>> dump(attrs.__code__) name: attrs argcount: 1 posonlyargcount: 0 @@ -80,8 +76,7 @@ ... 53 ... 0x53 -# TODO: RUSTPYTHON ->>> # dump(optimize_away.__code__) +>>> dump(optimize_away.__code__) name: optimize_away argcount: 0 posonlyargcount: 0 @@ -91,15 +86,14 @@ cellvars: () freevars: () nlocals: 0 -flags: 3 +flags: 67108867 consts: ("'doc string'", 'None') >>> def keywordonly_args(a,b,*,k1): ... return a,b,k1 ... -# TODO: RUSTPYTHON ->>> # dump(keywordonly_args.__code__) +>>> dump(keywordonly_args.__code__) name: keywordonly_args argcount: 2 posonlyargcount: 0 @@ -116,8 +110,7 @@ ... return a,b,c ... -# TODO: RUSTPYTHON ->>> # dump(posonly_args.__code__) +>>> dump(posonly_args.__code__) name: posonly_args argcount: 3 posonlyargcount: 2 @@ -130,8 +123,78 @@ flags: 3 consts: ('None',) +>>> def has_docstring(x: str): +... 'This is a one-line doc string' +... x += x +... x += "hello world" +... # co_flags should be 0x4000003 = 67108867 +... return x + +>>> dump(has_docstring.__code__) +name: has_docstring +argcount: 1 +posonlyargcount: 0 +kwonlyargcount: 0 +names: () +varnames: ('x',) +cellvars: () +freevars: () +nlocals: 1 +flags: 67108867 +consts: ("'This is a one-line doc string'", "'hello world'") + +>>> async def async_func_docstring(x: str, y: str): +... "This is a docstring from async function" +... import asyncio +... await asyncio.sleep(1) +... # co_flags should be 0x4000083 = 67108995 +... return x + y + +>>> dump(async_func_docstring.__code__) +name: async_func_docstring +argcount: 2 +posonlyargcount: 0 +kwonlyargcount: 0 +names: ('asyncio', 'sleep') +varnames: ('x', 'y', 'asyncio') +cellvars: () +freevars: () +nlocals: 3 +flags: 67108995 +consts: ("'This is a docstring from async function'", 'None') + +>>> def no_docstring(x, y, z): +... return x + "hello" + y + z + "world" + +>>> dump(no_docstring.__code__) +name: no_docstring +argcount: 3 +posonlyargcount: 0 +kwonlyargcount: 0 +names: () +varnames: ('x', 'y', 'z') +cellvars: () +freevars: () +nlocals: 3 +flags: 3 +consts: ("'hello'", "'world'") + +>>> class class_with_docstring: +... '''This is a docstring for class''' +... '''This line is not docstring''' +... pass + +>>> print(class_with_docstring.__doc__) +This is a docstring for class + +>>> class class_without_docstring: +... pass + +>>> print(class_without_docstring.__doc__) +None """ +import copy import inspect import sys import threading @@ -147,10 +210,21 @@ ctypes = None from test.support import (cpython_only, check_impl_detail, requires_debug_ranges, - gc_collect) + gc_collect, Py_GIL_DISABLED, late_deletion) from test.support.script_helper import assert_python_ok -from test.support import threading_helper +from test.support import threading_helper, import_helper +from test.support.bytecode_helper import instructions_with_positions from opcode import opmap, opname +try: + from _testcapi import code_offset_to_line +except ModuleNotFoundError: + code_offset_to_line = None +try: + import _testinternalcapi +except ModuleNotFoundError: + _testinternalcapi = None +import test._code_definitions as defs + COPY_FREE_VARS = opmap['COPY_FREE_VARS'] @@ -176,11 +250,12 @@ def dump(co): def external_getitem(self, i): return f"Foreign getitem: {super().__getitem__(i)}" + class CodeTest(unittest.TestCase): @cpython_only def test_newempty(self): - import _testcapi + _testcapi = import_helper.import_module("_testcapi") co = _testcapi.code_newempty("filename", "funcname", 15) self.assertEqual(co.co_filename, "filename") self.assertEqual(co.co_name, "funcname") @@ -259,10 +334,11 @@ def func(): return x code = func.__code__ - # different co_name, co_varnames, co_consts + # Different co_name, co_varnames, co_consts. + # Must have the same number of constants and + # variables or we get crashes. def func2(): y = 2 - z = 3 return y code2 = func2.__code__ @@ -271,7 +347,7 @@ def func2(): ("co_posonlyargcount", 0), ("co_kwonlyargcount", 0), ("co_nlocals", 1), - ("co_stacksize", 0), + ("co_stacksize", 1), ("co_flags", code.co_flags | inspect.CO_COROUTINE), ("co_firstlineno", 100), ("co_code", code2.co_code), @@ -287,11 +363,17 @@ def func2(): with self.subTest(attr=attr, value=value): new_code = code.replace(**{attr: value}) self.assertEqual(getattr(new_code, attr), value) + new_code = copy.replace(code, **{attr: value}) + self.assertEqual(getattr(new_code, attr), value) new_code = code.replace(co_varnames=code2.co_varnames, co_nlocals=code2.co_nlocals) self.assertEqual(new_code.co_varnames, code2.co_varnames) self.assertEqual(new_code.co_nlocals, code2.co_nlocals) + new_code = copy.replace(code, co_varnames=code2.co_varnames, + co_nlocals=code2.co_nlocals) + self.assertEqual(new_code.co_varnames, code2.co_varnames) + self.assertEqual(new_code.co_nlocals, code2.co_nlocals) def test_nlocals_mismatch(self): def func(): @@ -330,8 +412,6 @@ def func(): with self.assertRaises(ValueError): co.replace(co_nlocals=co.co_nlocals + 1) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_shrinking_localsplus(self): # Check that PyCode_NewWithPosOnlyArgs resizes both # localsplusnames and localspluskinds, if an argument is a cell. @@ -347,7 +427,7 @@ def func(): new_code = code = func.__code__.replace(co_linetable=b'') self.assertEqual(list(new_code.co_lines()), []) - # TODO: RUSTPYTHON + # TODO: RUSTPYTHON; co_lnotab intentionally not implemented (deprecated since 3.12) @unittest.expectedFailure def test_co_lnotab_is_deprecated(self): # TODO: remove in 3.14 def func(): @@ -356,20 +436,75 @@ def func(): with self.assertWarns(DeprecationWarning): func.__code__.co_lnotab - # TODO: RUSTPYTHON + @unittest.skipIf(_testinternalcapi is None, '_testinternalcapi is missing') + def test_returns_only_none(self): + value = True + + def spam1(): + pass + def spam2(): + return + def spam3(): + return None + def spam4(): + if not value: + return + ... + def spam5(): + if not value: + return None + ... + lambda1 = (lambda: None) + for func in [ + spam1, + spam2, + spam3, + spam4, + spam5, + lambda1, + ]: + with self.subTest(func): + res = _testinternalcapi.code_returns_only_none(func.__code__) + self.assertTrue(res) + + def spam6(): + return True + def spam7(): + return value + def spam8(): + if value: + return None + return True + def spam9(): + if value: + return True + return None + lambda2 = (lambda: True) + for func in [ + spam6, + spam7, + spam8, + spam9, + lambda2, + ]: + with self.subTest(func): + res = _testinternalcapi.code_returns_only_none(func.__code__) + self.assertFalse(res) + + # TODO: RUSTPYTHON; replace() rejects invalid bytecodes for safety @unittest.expectedFailure def test_invalid_bytecode(self): def foo(): pass - # assert that opcode 229 is invalid - self.assertEqual(opname[229], '<229>') + # assert that opcode 127 is invalid + self.assertEqual(opname[127], '<127>') - # change first opcode to 0xeb (=229) + # change first opcode to 0x7f (=127) foo.__code__ = foo.__code__.replace( - co_code=b'\xe5' + foo.__code__.co_code[1:]) + co_code=b'\x7f' + foo.__code__.co_code[1:]) - msg = "unknown opcode 229" + msg = "unknown opcode 127" with self.assertRaisesRegex(SystemError, msg): foo() @@ -392,10 +527,8 @@ def test_co_positions_artificial_instructions(self): code = traceback.tb_frame.f_code artificial_instructions = [] - for instr, positions in zip( - dis.get_instructions(code, show_caches=True), - code.co_positions(), - strict=True + for instr, positions in instructions_with_positions( + dis.get_instructions(code), code.co_positions() ): # If any of the positions is None, then all have to # be None as well for the case above. There are still @@ -456,7 +589,7 @@ def f(): # co_positions behavior when info is missing. - # @requires_debug_ranges() + @requires_debug_ranges() def test_co_positions_empty_linetable(self): def func(): x = 1 @@ -513,6 +646,533 @@ def test_code_hash_uses_bytecode(self): self.assertNotEqual(c, c1) self.assertNotEqual(hash(c), hash(c1)) + @cpython_only + def test_code_equal_with_instrumentation(self): + """ GH-109052 + + Make sure the instrumentation doesn't affect the code equality + The validity of this test relies on the fact that "x is x" and + "x in x" have only one different instruction and the instructions + have the same argument. + + """ + code1 = compile("x is x", "example.py", "eval") + code2 = compile("x in x", "example.py", "eval") + sys._getframe().f_trace_opcodes = True + sys.settrace(lambda *args: None) + exec(code1, {'x': []}) + exec(code2, {'x': []}) + self.assertNotEqual(code1, code2) + sys.settrace(None) + + @unittest.skipIf(_testinternalcapi is None, "missing _testinternalcapi") + def test_local_kinds(self): + CO_FAST_ARG_POS = 0x02 + CO_FAST_ARG_KW = 0x04 + CO_FAST_ARG_VAR = 0x08 + CO_FAST_HIDDEN = 0x10 + CO_FAST_LOCAL = 0x20 + CO_FAST_CELL = 0x40 + CO_FAST_FREE = 0x80 + + POSONLY = CO_FAST_LOCAL | CO_FAST_ARG_POS + POSORKW = CO_FAST_LOCAL | CO_FAST_ARG_POS | CO_FAST_ARG_KW + KWONLY = CO_FAST_LOCAL | CO_FAST_ARG_KW + VARARGS = CO_FAST_LOCAL | CO_FAST_ARG_VAR | CO_FAST_ARG_POS + VARKWARGS = CO_FAST_LOCAL | CO_FAST_ARG_VAR | CO_FAST_ARG_KW + + funcs = { + defs.simple_script: {}, + defs.complex_script: { + 'obj': CO_FAST_LOCAL, + 'pickle': CO_FAST_LOCAL, + 'spam_minimal': CO_FAST_LOCAL, + 'data': CO_FAST_LOCAL, + 'res': CO_FAST_LOCAL, + }, + defs.script_with_globals: { + 'obj1': CO_FAST_LOCAL, + 'obj2': CO_FAST_LOCAL, + }, + defs.script_with_explicit_empty_return: {}, + defs.script_with_return: {}, + defs.spam_minimal: {}, + defs.spam_with_builtins: { + 'x': CO_FAST_LOCAL, + 'values': CO_FAST_LOCAL, + 'checks': CO_FAST_LOCAL, + 'res': CO_FAST_LOCAL, + }, + defs.spam_with_globals_and_builtins: { + 'func1': CO_FAST_LOCAL, + 'func2': CO_FAST_LOCAL, + 'funcs': CO_FAST_LOCAL, + 'checks': CO_FAST_LOCAL, + 'res': CO_FAST_LOCAL, + }, + defs.spam_with_global_and_attr_same_name: {}, + defs.spam_full_args: { + 'a': POSONLY, + 'b': POSONLY, + 'c': POSORKW, + 'd': POSORKW, + 'e': KWONLY, + 'f': KWONLY, + 'args': VARARGS, + 'kwargs': VARKWARGS, + }, + defs.spam_full_args_with_defaults: { + 'a': POSONLY, + 'b': POSONLY, + 'c': POSORKW, + 'd': POSORKW, + 'e': KWONLY, + 'f': KWONLY, + 'args': VARARGS, + 'kwargs': VARKWARGS, + }, + defs.spam_args_attrs_and_builtins: { + 'a': POSONLY, + 'b': POSONLY, + 'c': POSORKW, + 'd': POSORKW, + 'e': KWONLY, + 'f': KWONLY, + 'args': VARARGS, + 'kwargs': VARKWARGS, + }, + defs.spam_returns_arg: { + 'x': POSORKW, + }, + defs.spam_raises: {}, + defs.spam_with_inner_not_closure: { + 'eggs': CO_FAST_LOCAL, + }, + defs.spam_with_inner_closure: { + 'x': CO_FAST_CELL, + 'eggs': CO_FAST_LOCAL, + }, + defs.spam_annotated: { + 'a': POSORKW, + 'b': POSORKW, + 'c': POSORKW, + }, + defs.spam_full: { + 'a': POSONLY, + 'b': POSONLY, + 'c': POSORKW, + 'd': POSORKW, + 'e': KWONLY, + 'f': KWONLY, + 'args': VARARGS, + 'kwargs': VARKWARGS, + 'x': CO_FAST_LOCAL, + 'y': CO_FAST_LOCAL, + 'z': CO_FAST_LOCAL, + 'extras': CO_FAST_LOCAL, + }, + defs.spam: { + 'x': POSORKW, + }, + defs.spam_N: { + 'x': POSORKW, + 'eggs_nested': CO_FAST_LOCAL, + }, + defs.spam_C: { + 'x': POSORKW | CO_FAST_CELL, + 'a': CO_FAST_CELL, + 'eggs_closure': CO_FAST_LOCAL, + }, + defs.spam_NN: { + 'x': POSORKW, + 'eggs_nested_N': CO_FAST_LOCAL, + }, + defs.spam_NC: { + 'x': POSORKW | CO_FAST_CELL, + 'a': CO_FAST_CELL, + 'eggs_nested_C': CO_FAST_LOCAL, + }, + defs.spam_CN: { + 'x': POSORKW | CO_FAST_CELL, + 'a': CO_FAST_CELL, + 'eggs_closure_N': CO_FAST_LOCAL, + }, + defs.spam_CC: { + 'x': POSORKW | CO_FAST_CELL, + 'a': CO_FAST_CELL, + 'eggs_closure_C': CO_FAST_LOCAL, + }, + defs.eggs_nested: { + 'y': POSORKW, + }, + defs.eggs_closure: { + 'y': POSORKW, + 'x': CO_FAST_FREE, + 'a': CO_FAST_FREE, + }, + defs.eggs_nested_N: { + 'y': POSORKW, + 'ham_nested': CO_FAST_LOCAL, + }, + defs.eggs_nested_C: { + 'y': POSORKW | CO_FAST_CELL, + 'x': CO_FAST_FREE, + 'a': CO_FAST_FREE, + 'ham_closure': CO_FAST_LOCAL, + }, + defs.eggs_closure_N: { + 'y': POSORKW, + 'x': CO_FAST_FREE, + 'a': CO_FAST_FREE, + 'ham_C_nested': CO_FAST_LOCAL, + }, + defs.eggs_closure_C: { + 'y': POSORKW | CO_FAST_CELL, + 'b': CO_FAST_CELL, + 'x': CO_FAST_FREE, + 'a': CO_FAST_FREE, + 'ham_C_closure': CO_FAST_LOCAL, + }, + defs.ham_nested: { + 'z': POSORKW, + }, + defs.ham_closure: { + 'z': POSORKW, + 'y': CO_FAST_FREE, + 'x': CO_FAST_FREE, + 'a': CO_FAST_FREE, + }, + defs.ham_C_nested: { + 'z': POSORKW, + }, + defs.ham_C_closure: { + 'z': POSORKW, + 'y': CO_FAST_FREE, + 'b': CO_FAST_FREE, + 'x': CO_FAST_FREE, + 'a': CO_FAST_FREE, + }, + } + assert len(funcs) == len(defs.FUNCTIONS) + for func in defs.FUNCTIONS: + with self.subTest(func): + expected = funcs[func] + kinds = _testinternalcapi.get_co_localskinds(func.__code__) + self.assertEqual(kinds, expected) + + @unittest.skipIf(_testinternalcapi is None, "missing _testinternalcapi") + def test_var_counts(self): + self.maxDiff = None + def new_var_counts(*, + posonly=0, + posorkw=0, + kwonly=0, + varargs=0, + varkwargs=0, + purelocals=0, + argcells=0, + othercells=0, + freevars=0, + globalvars=0, + attrs=0, + unknown=0, + ): + nargvars = posonly + posorkw + kwonly + varargs + varkwargs + nlocals = nargvars + purelocals + othercells + if isinstance(globalvars, int): + globalvars = { + 'total': globalvars, + 'numglobal': 0, + 'numbuiltin': 0, + 'numunknown': globalvars, + } + else: + g_numunknown = 0 + if isinstance(globalvars, dict): + numglobal = globalvars['numglobal'] + numbuiltin = globalvars['numbuiltin'] + size = 2 + if 'numunknown' in globalvars: + g_numunknown = globalvars['numunknown'] + size += 1 + assert len(globalvars) == size, globalvars + else: + assert not isinstance(globalvars, str), repr(globalvars) + try: + numglobal, numbuiltin = globalvars + except ValueError: + numglobal, numbuiltin, g_numunknown = globalvars + globalvars = { + 'total': numglobal + numbuiltin + g_numunknown, + 'numglobal': numglobal, + 'numbuiltin': numbuiltin, + 'numunknown': g_numunknown, + } + unbound = globalvars['total'] + attrs + unknown + return { + 'total': nlocals + freevars + unbound, + 'locals': { + 'total': nlocals, + 'args': { + 'total': nargvars, + 'numposonly': posonly, + 'numposorkw': posorkw, + 'numkwonly': kwonly, + 'varargs': varargs, + 'varkwargs': varkwargs, + }, + 'numpure': purelocals, + 'cells': { + 'total': argcells + othercells, + 'numargs': argcells, + 'numothers': othercells, + }, + 'hidden': { + 'total': 0, + 'numpure': 0, + 'numcells': 0, + }, + }, + 'numfree': freevars, + 'unbound': { + 'total': unbound, + 'globals': globalvars, + 'numattrs': attrs, + 'numunknown': unknown, + }, + } + + funcs = { + defs.simple_script: new_var_counts(), + defs.complex_script: new_var_counts( + purelocals=5, + globalvars=1, + attrs=2, + ), + defs.script_with_globals: new_var_counts( + purelocals=2, + globalvars=1, + ), + defs.script_with_explicit_empty_return: new_var_counts(), + defs.script_with_return: new_var_counts(), + defs.spam_minimal: new_var_counts(), + defs.spam_minimal: new_var_counts(), + defs.spam_with_builtins: new_var_counts( + purelocals=4, + globalvars=4, + ), + defs.spam_with_globals_and_builtins: new_var_counts( + purelocals=5, + globalvars=6, + ), + defs.spam_with_global_and_attr_same_name: new_var_counts( + globalvars=2, + attrs=1, + ), + defs.spam_full_args: new_var_counts( + posonly=2, + posorkw=2, + kwonly=2, + varargs=1, + varkwargs=1, + ), + defs.spam_full_args_with_defaults: new_var_counts( + posonly=2, + posorkw=2, + kwonly=2, + varargs=1, + varkwargs=1, + ), + defs.spam_args_attrs_and_builtins: new_var_counts( + posonly=2, + posorkw=2, + kwonly=2, + varargs=1, + varkwargs=1, + attrs=1, + ), + defs.spam_returns_arg: new_var_counts( + posorkw=1, + ), + defs.spam_raises: new_var_counts( + globalvars=1, + ), + defs.spam_with_inner_not_closure: new_var_counts( + purelocals=1, + ), + defs.spam_with_inner_closure: new_var_counts( + othercells=1, + purelocals=1, + ), + defs.spam_annotated: new_var_counts( + posorkw=3, + ), + defs.spam_full: new_var_counts( + posonly=2, + posorkw=2, + kwonly=2, + varargs=1, + varkwargs=1, + purelocals=4, + globalvars=3, + attrs=1, + ), + defs.spam: new_var_counts( + posorkw=1, + ), + defs.spam_N: new_var_counts( + posorkw=1, + purelocals=1, + ), + defs.spam_C: new_var_counts( + posorkw=1, + purelocals=1, + argcells=1, + othercells=1, + ), + defs.spam_NN: new_var_counts( + posorkw=1, + purelocals=1, + ), + defs.spam_NC: new_var_counts( + posorkw=1, + purelocals=1, + argcells=1, + othercells=1, + ), + defs.spam_CN: new_var_counts( + posorkw=1, + purelocals=1, + argcells=1, + othercells=1, + ), + defs.spam_CC: new_var_counts( + posorkw=1, + purelocals=1, + argcells=1, + othercells=1, + ), + defs.eggs_nested: new_var_counts( + posorkw=1, + ), + defs.eggs_closure: new_var_counts( + posorkw=1, + freevars=2, + ), + defs.eggs_nested_N: new_var_counts( + posorkw=1, + purelocals=1, + ), + defs.eggs_nested_C: new_var_counts( + posorkw=1, + purelocals=1, + argcells=1, + freevars=2, + ), + defs.eggs_closure_N: new_var_counts( + posorkw=1, + purelocals=1, + freevars=2, + ), + defs.eggs_closure_C: new_var_counts( + posorkw=1, + purelocals=1, + argcells=1, + othercells=1, + freevars=2, + ), + defs.ham_nested: new_var_counts( + posorkw=1, + ), + defs.ham_closure: new_var_counts( + posorkw=1, + freevars=3, + ), + defs.ham_C_nested: new_var_counts( + posorkw=1, + ), + defs.ham_C_closure: new_var_counts( + posorkw=1, + freevars=4, + ), + } + assert len(funcs) == len(defs.FUNCTIONS), (len(funcs), len(defs.FUNCTIONS)) + for func in defs.FUNCTIONS: + with self.subTest(func): + expected = funcs[func] + counts = _testinternalcapi.get_code_var_counts(func.__code__) + self.assertEqual(counts, expected) + + func = defs.spam_with_globals_and_builtins + with self.subTest(f'{func} code'): + expected = new_var_counts( + purelocals=5, + globalvars=6, + ) + counts = _testinternalcapi.get_code_var_counts(func.__code__) + self.assertEqual(counts, expected) + + with self.subTest(f'{func} with own globals and builtins'): + expected = new_var_counts( + purelocals=5, + globalvars=(2, 4), + ) + counts = _testinternalcapi.get_code_var_counts(func) + self.assertEqual(counts, expected) + + with self.subTest(f'{func} without globals'): + expected = new_var_counts( + purelocals=5, + globalvars=(0, 4, 2), + ) + counts = _testinternalcapi.get_code_var_counts(func, globalsns={}) + self.assertEqual(counts, expected) + + with self.subTest(f'{func} without both'): + expected = new_var_counts( + purelocals=5, + globalvars=6, + ) + counts = _testinternalcapi.get_code_var_counts(func, globalsns={}, + builtinsns={}) + self.assertEqual(counts, expected) + + with self.subTest(f'{func} without builtins'): + expected = new_var_counts( + purelocals=5, + globalvars=(2, 0, 4), + ) + counts = _testinternalcapi.get_code_var_counts(func, builtinsns={}) + self.assertEqual(counts, expected) + + @unittest.skipIf(_testinternalcapi is None, "missing _testinternalcapi") + def test_stateless(self): + self.maxDiff = None + + STATELESS_FUNCTIONS = [ + *defs.STATELESS_FUNCTIONS, + # stateless with defaults + defs.spam_full_args_with_defaults, + ] + + for func in defs.STATELESS_CODE: + with self.subTest((func, '(code)')): + _testinternalcapi.verify_stateless_code(func.__code__) + for func in STATELESS_FUNCTIONS: + with self.subTest((func, '(func)')): + _testinternalcapi.verify_stateless_code(func) + + for func in defs.FUNCTIONS: + if func not in defs.STATELESS_CODE: + with self.subTest((func, '(code)')): + with self.assertRaises(Exception): + _testinternalcapi.verify_stateless_code(func.__code__) + + if func not in STATELESS_FUNCTIONS: + with self.subTest((func, '(func)')): + with self.assertRaises(Exception): + _testinternalcapi.verify_stateless_code(func) + def isinterned(s): return s is sys.intern(('_' + s + '_')[1:-1]) @@ -559,11 +1219,47 @@ def f(a='str_value'): self.assertIsInterned(f()) @cpython_only + @unittest.skipIf(Py_GIL_DISABLED, "free-threaded build interns all string constants") def test_interned_string_with_null(self): co = compile(r'res = "str\0value!"', '?', 'exec') v = self.find_const(co.co_consts, 'str\0value!') self.assertIsNotInterned(v) + @cpython_only + @unittest.skipUnless(Py_GIL_DISABLED, "does not intern all constants") + def test_interned_constants(self): + # compile separately to avoid compile time de-duping + + globals = {} + exec(textwrap.dedent(""" + def func1(): + return (0.0, (1, 2, "hello")) + """), globals) + + exec(textwrap.dedent(""" + def func2(): + return (0.0, (1, 2, "hello")) + """), globals) + + self.assertTrue(globals["func1"]() is globals["func2"]()) + + @cpython_only + def test_unusual_constants(self): + # gh-130851: Code objects constructed with constants that are not + # types generated by the bytecode compiler should not crash the + # interpreter. + class Unhashable: + def __hash__(self): + raise TypeError("unhashable type") + + class MyInt(int): + pass + + code = compile("a = 1", "", "exec") + code = code.replace(co_consts=(1, Unhashable(), MyInt(1), MyInt(1))) + self.assertIsInstance(code.co_consts[1], Unhashable) + self.assertEqual(code.co_consts[2], code.co_consts[3]) + class CodeWeakRefTest(unittest.TestCase): @@ -750,7 +1446,7 @@ def f(): co_code=bytes( [ dis.opmap["RESUME"], 0, - dis.opmap["LOAD_ASSERTION_ERROR"], 0, + dis.opmap["LOAD_COMMON_CONSTANT"], 0, dis.opmap["RAISE_VARARGS"], 1, ] ), @@ -769,6 +1465,81 @@ def f(): 3 * [(42, 42, None, None)], ) + @cpython_only + def test_docstring_under_o2(self): + code = textwrap.dedent(''' + def has_docstring(x, y): + """This is a first-line doc string""" + """This is a second-line doc string""" + a = x + y + b = x - y + return a, b + + + def no_docstring(x): + def g(y): + return x + y + return g + + + async def async_func(): + """asynf function doc string""" + pass + + + for func in [has_docstring, no_docstring(4), async_func]: + assert(func.__doc__ is None) + ''') + + rc, out, err = assert_python_ok('-OO', '-c', code) + + @unittest.skipUnless(code_offset_to_line, '_testcapi required') + def test_co_branches(self): + + def get_line_branches(func): + code = func.__code__ + base = code.co_firstlineno + return [ + ( + code_offset_to_line(code, src) - base, + code_offset_to_line(code, left) - base, + code_offset_to_line(code, right) - base + ) for (src, left, right) in + code.co_branches() + ] + + def simple(x): + if x: + A + else: + B + + self.assertEqual( + get_line_branches(simple), + [(1,2,4)]) + + def with_extended_args(x): + if x: + A.x; A.x; A.x; A.x; A.x; A.x; + A.x; A.x; A.x; A.x; A.x; A.x; + A.x; A.x; A.x; A.x; A.x; A.x; + A.x; A.x; A.x; A.x; A.x; A.x; + A.x; A.x; A.x; A.x; A.x; A.x; + else: + B + + self.assertEqual( + get_line_branches(with_extended_args), + [(1,2,8)]) + + async def afunc(): + async for letter in async_iter1: + 2 + 3 + + self.assertEqual( + get_line_branches(afunc), + [(1,1,3)]) if check_impl_detail(cpython=True) and ctypes is not None: py = ctypes.pythonapi @@ -794,6 +1565,11 @@ def myfree(ptr): FREE_FUNC = freefunc(myfree) FREE_INDEX = RequestCodeExtraIndex(FREE_FUNC) + # Make sure myfree sticks around at least as long as the interpreter, + # since we (currently) can't unregister the function and leaving a + # dangling pointer will cause a crash on deallocation of code objects if + # something else uses co_extras, like test_capi.test_misc. + late_deletion(myfree) class CoExtra(unittest.TestCase): def get_func(self): @@ -824,6 +1600,7 @@ def test_free_called(self): SetExtra(f.__code__, FREE_INDEX, ctypes.c_voidp(100)) del f + gc_collect() # For free-threaded build self.assertEqual(LAST_FREED, 100) def test_get_set(self): @@ -854,13 +1631,19 @@ def __init__(self, f, test): self.test = test def run(self): del self.f - self.test.assertEqual(LAST_FREED, 500) + gc_collect() + # gh-117683: In the free-threaded build, the code object's + # destructor may still be running concurrently in the main + # thread. + if not Py_GIL_DISABLED: + self.test.assertEqual(LAST_FREED, 500) SetExtra(f.__code__, FREE_INDEX, ctypes.c_voidp(500)) tt = ThreadTest(f, self) del f tt.start() tt.join() + gc_collect() # For free-threaded build self.assertEqual(LAST_FREED, 500) diff --git a/Lib/test/test_code_module.py b/Lib/test/test_code_module.py index 4d5463e6db3..39d85d46274 100644 --- a/Lib/test/test_code_module.py +++ b/Lib/test/test_code_module.py @@ -39,19 +39,47 @@ def setUp(self): self.mock_sys() def test_ps1(self): - self.infunc.side_effect = EOFError('Finished') + self.infunc.side_effect = [ + "import code", + "code.sys.ps1", + EOFError('Finished') + ] self.console.interact() - self.assertEqual(self.sysmod.ps1, '>>> ') + output = ''.join(''.join(call[1]) for call in self.stdout.method_calls) + self.assertIn('>>> ', output) + self.assertNotHasAttr(self.sysmod, 'ps1') + + self.infunc.side_effect = [ + "import code", + "code.sys.ps1", + EOFError('Finished') + ] self.sysmod.ps1 = 'custom1> ' self.console.interact() + output = ''.join(''.join(call[1]) for call in self.stdout.method_calls) + self.assertIn('custom1> ', output) self.assertEqual(self.sysmod.ps1, 'custom1> ') def test_ps2(self): - self.infunc.side_effect = EOFError('Finished') + self.infunc.side_effect = [ + "import code", + "code.sys.ps2", + EOFError('Finished') + ] self.console.interact() - self.assertEqual(self.sysmod.ps2, '... ') + output = ''.join(''.join(call[1]) for call in self.stdout.method_calls) + self.assertIn('... ', output) + self.assertNotHasAttr(self.sysmod, 'ps2') + + self.infunc.side_effect = [ + "import code", + "code.sys.ps2", + EOFError('Finished') + ] self.sysmod.ps2 = 'custom2> ' self.console.interact() + output = ''.join(''.join(call[1]) for call in self.stdout.method_calls) + self.assertIn('custom2> ', output) self.assertEqual(self.sysmod.ps2, 'custom2> ') def test_console_stderr(self): @@ -63,9 +91,6 @@ def test_console_stderr(self): else: raise AssertionError("no console stdout") - # TODO: RUSTPYTHON - # AssertionError: Lists differ: [' F[27 chars] x = ?', ' ^', 'SyntaxError: got unexpected token ?'] != [' F[27 chars] x = ?', ' ^', 'SyntaxError: invalid syntax'] - @unittest.expectedFailure def test_syntax_error(self): self.infunc.side_effect = ["def f():", " x = ?", @@ -86,9 +111,7 @@ def test_syntax_error(self): self.assertIsNone(self.sysmod.last_value.__traceback__) self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) - # TODO: RUSTPYTHON - # AssertionError: Lists differ: [' F[15 chars], line 1', ' 1', 'IndentationError: unexpected indentation'] != [' F[15 chars], line 1', ' 1', 'IndentationError: unexpected indent'] - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; - 'IndentationError: unexpected indentation'] def test_indentation_error(self): self.infunc.side_effect = [" 1", EOFError('Finished')] self.console.interact() @@ -105,16 +128,14 @@ def test_indentation_error(self): self.assertIsNone(self.sysmod.last_value.__traceback__) self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) - # TODO: RUSTPYTHON - # AssertionError: False is not true : UnicodeDecodeError: invalid utf-8 sequence of 1 bytes from index 1 - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 'UnicodeDecodeError: invalid utf-8 sequence of 1 bytes from index 1\n\nnow exiti [truncated]... doesn't start with 'UnicodeEncodeError: ' def test_unicode_error(self): self.infunc.side_effect = ["'\ud800'", EOFError('Finished')] self.console.interact() output = ''.join(''.join(call[1]) for call in self.stderr.method_calls) output = output[output.index('(InteractiveConsole)'):] output = output[output.index('\n') + 1:] - self.assertTrue(output.startswith('UnicodeEncodeError: '), output) + self.assertStartsWith(output, 'UnicodeEncodeError: ') self.assertIs(self.sysmod.last_type, UnicodeEncodeError) self.assertIs(type(self.sysmod.last_value), UnicodeEncodeError) self.assertIsNone(self.sysmod.last_traceback) @@ -144,9 +165,6 @@ def test_sysexcepthook(self): ' File "", line 2, in f\n', 'ValueError: BOOM!\n']) - # TODO: RUSTPYTHON - # AssertionError: Lists differ: [' F[35 chars]= ?\n', ' ^\n', 'SyntaxError: got unexpected token ?\n'] != [' F[35 chars]= ?\n', ' ^\n', 'SyntaxError: invalid syntax\n'] - @unittest.expectedFailure def test_sysexcepthook_syntax_error(self): self.infunc.side_effect = ["def f():", " x = ?", @@ -170,9 +188,7 @@ def test_sysexcepthook_syntax_error(self): ' ^\n', 'SyntaxError: invalid syntax\n']) - # TODO: RUSTPYTHON - # AssertionError: Lists differ: [' F[21 chars] 1\n', ' 1\n', 'IndentationError: unexpected indentation\n'] != [' F[21 chars] 1\n', ' 1\n', 'IndentationError: unexpected indent\n'] - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; + 'IndentationError: unexpected indent\n'] def test_sysexcepthook_indentation_error(self): self.infunc.side_effect = [" 1", EOFError('Finished')] hook = mock.Mock() @@ -267,7 +283,6 @@ def test_exit_msg(self): self.assertEqual(err_msg, ['write', (expected,), {}]) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_cause_tb(self): self.infunc.side_effect = ["raise ValueError('') from AttributeError", EOFError('Finished')] diff --git a/Lib/test/test_codeccallbacks.py b/Lib/test/test_codeccallbacks.py index 9ca02cea351..763146c94fc 100644 --- a/Lib/test/test_codeccallbacks.py +++ b/Lib/test/test_codeccallbacks.py @@ -281,8 +281,6 @@ def handler2(exc): b"g[<252><223>]" ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_longstrings(self): # test long strings to check for memory overflow problems errors = [ "strict", "ignore", "replace", "xmlcharrefreplace", @@ -684,8 +682,6 @@ def test_badandgoodsurrogateescapeexceptions(self): ("\udc80", 2) ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_badandgoodsurrogatepassexceptions(self): surrogatepass_errors = codecs.lookup_error('surrogatepass') # "surrogatepass" complains about a non-exception passed in diff --git a/Lib/test/test_codecencodings_cn.py b/Lib/test/test_codecencodings_cn.py new file mode 100644 index 00000000000..af32f624d81 --- /dev/null +++ b/Lib/test/test_codecencodings_cn.py @@ -0,0 +1,100 @@ +# +# test_codecencodings_cn.py +# Codec encoding tests for PRC encodings. +# + +from test import multibytecodec_support +import unittest + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: gb2312") +class Test_GB2312(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'gb2312' + tstring = multibytecodec_support.load_teststring('gb2312') + codectests = ( + # invalid bytes + (b"abc\x81\x81\xc1\xc4", "strict", None), + (b"abc\xc8", "strict", None), + (b"abc\x81\x81\xc1\xc4", "replace", "abc\ufffd\ufffd\u804a"), + (b"abc\x81\x81\xc1\xc4\xc8", "replace", "abc\ufffd\ufffd\u804a\ufffd"), + (b"abc\x81\x81\xc1\xc4", "ignore", "abc\u804a"), + (b"\xc1\x64", "strict", None), + ) + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: gbk") +class Test_GBK(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'gbk' + tstring = multibytecodec_support.load_teststring('gbk') + codectests = ( + # invalid bytes + (b"abc\x80\x80\xc1\xc4", "strict", None), + (b"abc\xc8", "strict", None), + (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\ufffd\u804a"), + (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\ufffd\u804a\ufffd"), + (b"abc\x80\x80\xc1\xc4", "ignore", "abc\u804a"), + (b"\x83\x34\x83\x31", "strict", None), + ("\u30fb", "strict", None), + ) + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: gb18030") +class Test_GB18030(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'gb18030' + tstring = multibytecodec_support.load_teststring('gb18030') + codectests = ( + # invalid bytes + (b"abc\x80\x80\xc1\xc4", "strict", None), + (b"abc\xc8", "strict", None), + (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\ufffd\u804a"), + (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\ufffd\u804a\ufffd"), + (b"abc\x80\x80\xc1\xc4", "ignore", "abc\u804a"), + (b"abc\x84\x39\x84\x39\xc1\xc4", "replace", "abc\ufffd9\ufffd9\u804a"), + ("\u30fb", "strict", b"\x819\xa79"), + (b"abc\x84\x32\x80\x80def", "replace", 'abc\ufffd2\ufffd\ufffddef'), + (b"abc\x81\x30\x81\x30def", "strict", 'abc\x80def'), + (b"abc\x86\x30\x81\x30def", "replace", 'abc\ufffd0\ufffd0def'), + # issue29990 + (b"\xff\x30\x81\x30", "strict", None), + (b"\x81\x30\xff\x30", "strict", None), + (b"abc\x81\x39\xff\x39\xc1\xc4", "replace", "abc\ufffd\x39\ufffd\x39\u804a"), + (b"abc\xab\x36\xff\x30def", "replace", 'abc\ufffd\x36\ufffd\x30def'), + (b"abc\xbf\x38\xff\x32\xc1\xc4", "ignore", "abc\x38\x32\u804a"), + ) + has_iso10646 = True + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: hz") +class Test_HZ(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'hz' + tstring = multibytecodec_support.load_teststring('hz') + codectests = ( + # test '~\n' (3 lines) + (b'This sentence is in ASCII.\n' + b'The next sentence is in GB.~{<:Ky2;S{#,~}~\n' + b'~{NpJ)l6HK!#~}Bye.\n', + 'strict', + 'This sentence is in ASCII.\n' + 'The next sentence is in GB.' + '\u5df1\u6240\u4e0d\u6b32\uff0c\u52ff\u65bd\u65bc\u4eba\u3002' + 'Bye.\n'), + # test '~\n' (4 lines) + (b'This sentence is in ASCII.\n' + b'The next sentence is in GB.~\n' + b'~{<:Ky2;S{#,NpJ)l6HK!#~}~\n' + b'Bye.\n', + 'strict', + 'This sentence is in ASCII.\n' + 'The next sentence is in GB.' + '\u5df1\u6240\u4e0d\u6b32\uff0c\u52ff\u65bd\u65bc\u4eba\u3002' + 'Bye.\n'), + # invalid bytes + (b'ab~cd', 'replace', 'ab\uFFFDcd'), + (b'ab\xffcd', 'replace', 'ab\uFFFDcd'), + (b'ab~{\x81\x81\x41\x44~}cd', 'replace', 'ab\uFFFD\uFFFD\u804Acd'), + (b'ab~{\x41\x44~}cd', 'replace', 'ab\u804Acd'), + (b"ab~{\x79\x79\x41\x44~}cd", "replace", "ab\ufffd\ufffd\u804acd"), + # issue 30003 + ('ab~cd', 'strict', b'ab~~cd'), # escape ~ + (b'~{Dc~~:C~}', 'strict', None), # ~~ only in ASCII mode + (b'~{Dc~\n:C~}', 'strict', None), # ~\n only in ASCII mode + ) + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_codecencodings_hk.py b/Lib/test/test_codecencodings_hk.py new file mode 100644 index 00000000000..b64d19bca91 --- /dev/null +++ b/Lib/test/test_codecencodings_hk.py @@ -0,0 +1,23 @@ +# +# test_codecencodings_hk.py +# Codec encoding tests for HongKong encodings. +# + +from test import multibytecodec_support +import unittest + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: big5hkscs") +class Test_Big5HKSCS(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'big5hkscs' + tstring = multibytecodec_support.load_teststring('big5hkscs') + codectests = ( + # invalid bytes + (b"abc\x80\x80\xc1\xc4", "strict", None), + (b"abc\xc8", "strict", None), + (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\ufffd\u8b10"), + (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\ufffd\u8b10\ufffd"), + (b"abc\x80\x80\xc1\xc4", "ignore", "abc\u8b10"), + ) + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_codecencodings_iso2022.py b/Lib/test/test_codecencodings_iso2022.py new file mode 100644 index 00000000000..fe97aa6977e --- /dev/null +++ b/Lib/test/test_codecencodings_iso2022.py @@ -0,0 +1,92 @@ +# Codec encoding tests for ISO 2022 encodings. + +from test import multibytecodec_support +import unittest + +COMMON_CODEC_TESTS = ( + # invalid bytes + (b'ab\xFFcd', 'replace', 'ab\uFFFDcd'), + (b'ab\x1Bdef', 'replace', 'ab\x1Bdef'), + (b'ab\x1B$def', 'replace', 'ab\uFFFD'), + ) + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: iso2022_jp") +class Test_ISO2022_JP(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'iso2022_jp' + tstring = multibytecodec_support.load_teststring('iso2022_jp') + codectests = COMMON_CODEC_TESTS + ( + (b'ab\x1BNdef', 'replace', 'ab\x1BNdef'), + ) + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: iso2022_jp_2") +class Test_ISO2022_JP2(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'iso2022_jp_2' + tstring = multibytecodec_support.load_teststring('iso2022_jp') + codectests = COMMON_CODEC_TESTS + ( + (b'ab\x1BNdef', 'replace', 'abdef'), + ) + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: iso2022_jp_3") +class Test_ISO2022_JP3(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'iso2022_jp_3' + tstring = multibytecodec_support.load_teststring('iso2022_jp') + codectests = COMMON_CODEC_TESTS + ( + (b'ab\x1BNdef', 'replace', 'ab\x1BNdef'), + (b'\x1B$(O\x2E\x23\x1B(B', 'strict', '\u3402' ), + (b'\x1B$(O\x2E\x22\x1B(B', 'strict', '\U0002000B' ), + (b'\x1B$(O\x24\x77\x1B(B', 'strict', '\u304B\u309A'), + (b'\x1B$(P\x21\x22\x1B(B', 'strict', '\u4E02' ), + (b'\x1B$(P\x7E\x76\x1B(B', 'strict', '\U0002A6B2' ), + ('\u3402', 'strict', b'\x1B$(O\x2E\x23\x1B(B'), + ('\U0002000B', 'strict', b'\x1B$(O\x2E\x22\x1B(B'), + ('\u304B\u309A', 'strict', b'\x1B$(O\x24\x77\x1B(B'), + ('\u4E02', 'strict', b'\x1B$(P\x21\x22\x1B(B'), + ('\U0002A6B2', 'strict', b'\x1B$(P\x7E\x76\x1B(B'), + (b'ab\x1B$(O\x2E\x21\x1B(Bdef', 'replace', 'ab\uFFFDdef'), + ('ab\u4FF1def', 'replace', b'ab?def'), + ) + xmlcharnametest = ( + '\xAB\u211C\xBB = \u2329\u1234\u232A', + b'\x1B$(O\x29\x28\x1B(Bℜ\x1B$(O\x29\x32\x1B(B = ⟨ሴ⟩' + ) + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: iso2022_jp_2004") +class Test_ISO2022_JP2004(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'iso2022_jp_2004' + tstring = multibytecodec_support.load_teststring('iso2022_jp') + codectests = COMMON_CODEC_TESTS + ( + (b'ab\x1BNdef', 'replace', 'ab\x1BNdef'), + (b'\x1B$(Q\x2E\x23\x1B(B', 'strict', '\u3402' ), + (b'\x1B$(Q\x2E\x22\x1B(B', 'strict', '\U0002000B' ), + (b'\x1B$(Q\x24\x77\x1B(B', 'strict', '\u304B\u309A'), + (b'\x1B$(P\x21\x22\x1B(B', 'strict', '\u4E02' ), + (b'\x1B$(P\x7E\x76\x1B(B', 'strict', '\U0002A6B2' ), + ('\u3402', 'strict', b'\x1B$(Q\x2E\x23\x1B(B'), + ('\U0002000B', 'strict', b'\x1B$(Q\x2E\x22\x1B(B'), + ('\u304B\u309A', 'strict', b'\x1B$(Q\x24\x77\x1B(B'), + ('\u4E02', 'strict', b'\x1B$(P\x21\x22\x1B(B'), + ('\U0002A6B2', 'strict', b'\x1B$(P\x7E\x76\x1B(B'), + (b'ab\x1B$(Q\x2E\x21\x1B(Bdef', 'replace', 'ab\u4FF1def'), + ('ab\u4FF1def', 'replace', b'ab\x1B$(Q\x2E\x21\x1B(Bdef'), + ) + xmlcharnametest = ( + '\xAB\u211C\xBB = \u2329\u1234\u232A', + b'\x1B$(Q\x29\x28\x1B(Bℜ\x1B$(Q\x29\x32\x1B(B = ⟨ሴ⟩' + ) + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: iso2022_kr") +class Test_ISO2022_KR(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'iso2022_kr' + tstring = multibytecodec_support.load_teststring('iso2022_kr') + codectests = COMMON_CODEC_TESTS + ( + (b'ab\x1BNdef', 'replace', 'ab\x1BNdef'), + ) + + # iso2022_kr.txt cannot be used to test "chunk coding": the escape + # sequence is only written on the first line + @unittest.skip('iso2022_kr.txt cannot be used to test "chunk coding"') + def test_chunkcoding(self): + pass + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_codecencodings_jp.py b/Lib/test/test_codecencodings_jp.py new file mode 100644 index 00000000000..f78ae229f84 --- /dev/null +++ b/Lib/test/test_codecencodings_jp.py @@ -0,0 +1,133 @@ +# +# test_codecencodings_jp.py +# Codec encoding tests for Japanese encodings. +# + +from test import multibytecodec_support +import unittest + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: cp932") +class Test_CP932(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'cp932' + tstring = multibytecodec_support.load_teststring('shift_jis') + codectests = ( + # invalid bytes + (b"abc\x81\x00\x81\x00\x82\x84", "strict", None), + (b"abc\xf8", "strict", None), + (b"abc\x81\x00\x82\x84", "replace", "abc\ufffd\x00\uff44"), + (b"abc\x81\x00\x82\x84\x88", "replace", "abc\ufffd\x00\uff44\ufffd"), + (b"abc\x81\x00\x82\x84", "ignore", "abc\x00\uff44"), + (b"ab\xEBxy", "replace", "ab\uFFFDxy"), + (b"ab\xF0\x39xy", "replace", "ab\uFFFD9xy"), + (b"ab\xEA\xF0xy", "replace", 'ab\ufffd\ue038y'), + # sjis vs cp932 + (b"\\\x7e", "replace", "\\\x7e"), + (b"\x81\x5f\x81\x61\x81\x7c", "replace", "\uff3c\u2225\uff0d"), + ) + +euc_commontests = ( + # invalid bytes + (b"abc\x80\x80\xc1\xc4", "strict", None), + (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\ufffd\u7956"), + (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\ufffd\u7956\ufffd"), + (b"abc\x80\x80\xc1\xc4", "ignore", "abc\u7956"), + (b"abc\xc8", "strict", None), + (b"abc\x8f\x83\x83", "replace", "abc\ufffd\ufffd\ufffd"), + (b"\x82\xFCxy", "replace", "\ufffd\ufffdxy"), + (b"\xc1\x64", "strict", None), + (b"\xa1\xc0", "strict", "\uff3c"), + (b"\xa1\xc0\\", "strict", "\uff3c\\"), + (b"\x8eXY", "replace", "\ufffdXY"), +) + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: euc_jis_2004") +class Test_EUC_JIS_2004(multibytecodec_support.TestBase, + unittest.TestCase): + encoding = 'euc_jis_2004' + tstring = multibytecodec_support.load_teststring('euc_jisx0213') + codectests = euc_commontests + xmlcharnametest = ( + "\xab\u211c\xbb = \u2329\u1234\u232a", + b"\xa9\xa8ℜ\xa9\xb2 = ⟨ሴ⟩" + ) + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: euc_jisx0213") +class Test_EUC_JISX0213(multibytecodec_support.TestBase, + unittest.TestCase): + encoding = 'euc_jisx0213' + tstring = multibytecodec_support.load_teststring('euc_jisx0213') + codectests = euc_commontests + xmlcharnametest = ( + "\xab\u211c\xbb = \u2329\u1234\u232a", + b"\xa9\xa8ℜ\xa9\xb2 = ⟨ሴ⟩" + ) + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: euc_jp") +class Test_EUC_JP_COMPAT(multibytecodec_support.TestBase, + unittest.TestCase): + encoding = 'euc_jp' + tstring = multibytecodec_support.load_teststring('euc_jp') + codectests = euc_commontests + ( + ("\xa5", "strict", b"\x5c"), + ("\u203e", "strict", b"\x7e"), + ) + +shiftjis_commonenctests = ( + (b"abc\x80\x80\x82\x84", "strict", None), + (b"abc\xf8", "strict", None), + (b"abc\x80\x80\x82\x84def", "ignore", "abc\uff44def"), +) + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: shift_jis") +class Test_SJIS_COMPAT(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'shift_jis' + tstring = multibytecodec_support.load_teststring('shift_jis') + codectests = shiftjis_commonenctests + ( + (b"abc\x80\x80\x82\x84", "replace", "abc\ufffd\ufffd\uff44"), + (b"abc\x80\x80\x82\x84\x88", "replace", "abc\ufffd\ufffd\uff44\ufffd"), + + (b"\\\x7e", "strict", "\\\x7e"), + (b"\x81\x5f\x81\x61\x81\x7c", "strict", "\uff3c\u2016\u2212"), + (b"abc\x81\x39", "replace", "abc\ufffd9"), + (b"abc\xEA\xFC", "replace", "abc\ufffd\ufffd"), + (b"abc\xFF\x58", "replace", "abc\ufffdX"), + ) + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: shift_jis_2004") +class Test_SJIS_2004(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'shift_jis_2004' + tstring = multibytecodec_support.load_teststring('shift_jis') + codectests = shiftjis_commonenctests + ( + (b"\\\x7e", "strict", "\xa5\u203e"), + (b"\x81\x5f\x81\x61\x81\x7c", "strict", "\\\u2016\u2212"), + (b"abc\xEA\xFC", "strict", "abc\u64bf"), + (b"\x81\x39xy", "replace", "\ufffd9xy"), + (b"\xFF\x58xy", "replace", "\ufffdXxy"), + (b"\x80\x80\x82\x84xy", "replace", "\ufffd\ufffd\uff44xy"), + (b"\x80\x80\x82\x84\x88xy", "replace", "\ufffd\ufffd\uff44\u5864y"), + (b"\xFC\xFBxy", "replace", '\ufffd\u95b4y'), + ) + xmlcharnametest = ( + "\xab\u211c\xbb = \u2329\u1234\u232a", + b"\x85Gℜ\x85Q = ⟨ሴ⟩" + ) + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: shift_jisx0213") +class Test_SJISX0213(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'shift_jisx0213' + tstring = multibytecodec_support.load_teststring('shift_jisx0213') + codectests = shiftjis_commonenctests + ( + (b"abc\x80\x80\x82\x84", "replace", "abc\ufffd\ufffd\uff44"), + (b"abc\x80\x80\x82\x84\x88", "replace", "abc\ufffd\ufffd\uff44\ufffd"), + + # sjis vs cp932 + (b"\\\x7e", "replace", "\xa5\u203e"), + (b"\x81\x5f\x81\x61\x81\x7c", "replace", "\x5c\u2016\u2212"), + ) + xmlcharnametest = ( + "\xab\u211c\xbb = \u2329\u1234\u232a", + b"\x85Gℜ\x85Q = ⟨ሴ⟩" + ) + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_codecencodings_kr.py b/Lib/test/test_codecencodings_kr.py new file mode 100644 index 00000000000..aee124007ae --- /dev/null +++ b/Lib/test/test_codecencodings_kr.py @@ -0,0 +1,72 @@ +# +# test_codecencodings_kr.py +# Codec encoding tests for ROK encodings. +# + +from test import multibytecodec_support +import unittest + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: cp949") +class Test_CP949(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'cp949' + tstring = multibytecodec_support.load_teststring('cp949') + codectests = ( + # invalid bytes + (b"abc\x80\x80\xc1\xc4", "strict", None), + (b"abc\xc8", "strict", None), + (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\ufffd\uc894"), + (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\ufffd\uc894\ufffd"), + (b"abc\x80\x80\xc1\xc4", "ignore", "abc\uc894"), + ) + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: euc_kr") +class Test_EUCKR(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'euc_kr' + tstring = multibytecodec_support.load_teststring('euc_kr') + codectests = ( + # invalid bytes + (b"abc\x80\x80\xc1\xc4", "strict", None), + (b"abc\xc8", "strict", None), + (b"abc\x80\x80\xc1\xc4", "replace", 'abc\ufffd\ufffd\uc894'), + (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\ufffd\uc894\ufffd"), + (b"abc\x80\x80\xc1\xc4", "ignore", "abc\uc894"), + + # composed make-up sequence errors + (b"\xa4\xd4", "strict", None), + (b"\xa4\xd4\xa4", "strict", None), + (b"\xa4\xd4\xa4\xb6", "strict", None), + (b"\xa4\xd4\xa4\xb6\xa4", "strict", None), + (b"\xa4\xd4\xa4\xb6\xa4\xd0", "strict", None), + (b"\xa4\xd4\xa4\xb6\xa4\xd0\xa4", "strict", None), + (b"\xa4\xd4\xa4\xb6\xa4\xd0\xa4\xd4", "strict", "\uc4d4"), + (b"\xa4\xd4\xa4\xb6\xa4\xd0\xa4\xd4x", "strict", "\uc4d4x"), + (b"a\xa4\xd4\xa4\xb6\xa4", "replace", 'a\ufffd'), + (b"\xa4\xd4\xa3\xb6\xa4\xd0\xa4\xd4", "strict", None), + (b"\xa4\xd4\xa4\xb6\xa3\xd0\xa4\xd4", "strict", None), + (b"\xa4\xd4\xa4\xb6\xa4\xd0\xa3\xd4", "strict", None), + (b"\xa4\xd4\xa4\xff\xa4\xd0\xa4\xd4", "replace", '\ufffd\u6e21\ufffd\u3160\ufffd'), + (b"\xa4\xd4\xa4\xb6\xa4\xff\xa4\xd4", "replace", '\ufffd\u6e21\ub544\ufffd\ufffd'), + (b"\xa4\xd4\xa4\xb6\xa4\xd0\xa4\xff", "replace", '\ufffd\u6e21\ub544\u572d\ufffd'), + (b"\xa4\xd4\xff\xa4\xd4\xa4\xb6\xa4\xd0\xa4\xd4", "replace", '\ufffd\ufffd\ufffd\uc4d4'), + (b"\xc1\xc4", "strict", "\uc894"), + ) + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: johab") +class Test_JOHAB(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'johab' + tstring = multibytecodec_support.load_teststring('johab') + codectests = ( + # invalid bytes + (b"abc\x80\x80\xc1\xc4", "strict", None), + (b"abc\xc8", "strict", None), + (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\ufffd\ucd27"), + (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\ufffd\ucd27\ufffd"), + (b"abc\x80\x80\xc1\xc4", "ignore", "abc\ucd27"), + (b"\xD8abc", "replace", "\uFFFDabc"), + (b"\xD8\xFFabc", "replace", "\uFFFD\uFFFDabc"), + (b"\x84bxy", "replace", "\uFFFDbxy"), + (b"\x8CBxy", "replace", "\uFFFDBxy"), + ) + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_codecencodings_tw.py b/Lib/test/test_codecencodings_tw.py new file mode 100644 index 00000000000..ca56b23234e --- /dev/null +++ b/Lib/test/test_codecencodings_tw.py @@ -0,0 +1,23 @@ +# +# test_codecencodings_tw.py +# Codec encoding tests for ROC encodings. +# + +from test import multibytecodec_support +import unittest + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: big5") +class Test_Big5(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'big5' + tstring = multibytecodec_support.load_teststring('big5') + codectests = ( + # invalid bytes + (b"abc\x80\x80\xc1\xc4", "strict", None), + (b"abc\xc8", "strict", None), + (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\ufffd\u8b10"), + (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\ufffd\u8b10\ufffd"), + (b"abc\x80\x80\xc1\xc4", "ignore", "abc\u8b10"), + ) + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_codecmaps_cn.py b/Lib/test/test_codecmaps_cn.py new file mode 100644 index 00000000000..de254c6f767 --- /dev/null +++ b/Lib/test/test_codecmaps_cn.py @@ -0,0 +1,38 @@ +# +# test_codecmaps_cn.py +# Codec mapping tests for PRC encodings +# + +from test import multibytecodec_support +import unittest + +class TestGB2312Map(multibytecodec_support.TestBase_Mapping, + unittest.TestCase): + encoding = 'gb2312' + mapfileurl = 'http://www.pythontest.net/unicode/EUC-CN.TXT' + + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: gb2312 + def test_mapping_file(self): + return super().test_mapping_file() + +class TestGBKMap(multibytecodec_support.TestBase_Mapping, + unittest.TestCase): + encoding = 'gbk' + mapfileurl = 'http://www.pythontest.net/unicode/CP936.TXT' + + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: gbk + def test_mapping_file(self): + return super().test_mapping_file() + +class TestGB18030Map(multibytecodec_support.TestBase_Mapping, + unittest.TestCase): + encoding = 'gb18030' + mapfileurl = 'http://www.pythontest.net/unicode/gb-18030-2000.xml' + + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: gb18030 + def test_mapping_file(self): + return super().test_mapping_file() + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_codecmaps_hk.py b/Lib/test/test_codecmaps_hk.py new file mode 100644 index 00000000000..f02cf486f76 --- /dev/null +++ b/Lib/test/test_codecmaps_hk.py @@ -0,0 +1,19 @@ +# +# test_codecmaps_hk.py +# Codec mapping tests for HongKong encodings +# + +from test import multibytecodec_support +import unittest + +class TestBig5HKSCSMap(multibytecodec_support.TestBase_Mapping, + unittest.TestCase): + encoding = 'big5hkscs' + mapfileurl = 'http://www.pythontest.net/unicode/BIG5HKSCS-2004.TXT' + + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: big5hkscs + def test_mapping_file(self): + return super().test_mapping_file() + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_codecmaps_jp.py b/Lib/test/test_codecmaps_jp.py new file mode 100644 index 00000000000..f2d52f99526 --- /dev/null +++ b/Lib/test/test_codecmaps_jp.py @@ -0,0 +1,84 @@ +# +# test_codecmaps_jp.py +# Codec mapping tests for Japanese encodings +# + +from test import multibytecodec_support +import unittest + +class TestCP932Map(multibytecodec_support.TestBase_Mapping, + unittest.TestCase): + encoding = 'cp932' + mapfileurl = 'http://www.pythontest.net/unicode/CP932.TXT' + supmaps = [ + (b'\x80', '\u0080'), + (b'\xa0', '\uf8f0'), + (b'\xfd', '\uf8f1'), + (b'\xfe', '\uf8f2'), + (b'\xff', '\uf8f3'), + ] + for i in range(0xa1, 0xe0): + supmaps.append((bytes([i]), chr(i+0xfec0))) + + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: cp932 + def test_mapping_file(self): + return super().test_mapping_file() + + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: cp932 + def test_mapping_supplemental(self): + return super().test_mapping_supplemental() + + +class TestEUCJPCOMPATMap(multibytecodec_support.TestBase_Mapping, + unittest.TestCase): + encoding = 'euc_jp' + mapfilename = 'EUC-JP.TXT' + mapfileurl = 'http://www.pythontest.net/unicode/EUC-JP.TXT' + + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: euc_jp + def test_mapping_file(self): + return super().test_mapping_file() + + +class TestSJISCOMPATMap(multibytecodec_support.TestBase_Mapping, + unittest.TestCase): + encoding = 'shift_jis' + mapfilename = 'SHIFTJIS.TXT' + mapfileurl = 'http://www.pythontest.net/unicode/SHIFTJIS.TXT' + pass_enctest = [ + (b'\x81_', '\\'), + ] + pass_dectest = [ + (b'\\', '\xa5'), + (b'~', '\u203e'), + (b'\x81_', '\\'), + ] + + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: shift_jis + def test_mapping_file(self): + return super().test_mapping_file() + +class TestEUCJISX0213Map(multibytecodec_support.TestBase_Mapping, + unittest.TestCase): + encoding = 'euc_jisx0213' + mapfilename = 'EUC-JISX0213.TXT' + mapfileurl = 'http://www.pythontest.net/unicode/EUC-JISX0213.TXT' + + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: euc_jisx0213 + def test_mapping_file(self): + return super().test_mapping_file() + + +class TestSJISX0213Map(multibytecodec_support.TestBase_Mapping, + unittest.TestCase): + encoding = 'shift_jisx0213' + mapfilename = 'SHIFT_JISX0213.TXT' + mapfileurl = 'http://www.pythontest.net/unicode/SHIFT_JISX0213.TXT' + + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: shift_jisx0213 + def test_mapping_file(self): + return super().test_mapping_file() + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_codecmaps_kr.py b/Lib/test/test_codecmaps_kr.py new file mode 100644 index 00000000000..b8376d36615 --- /dev/null +++ b/Lib/test/test_codecmaps_kr.py @@ -0,0 +1,49 @@ +# +# test_codecmaps_kr.py +# Codec mapping tests for ROK encodings +# + +from test import multibytecodec_support +import unittest + +class TestCP949Map(multibytecodec_support.TestBase_Mapping, + unittest.TestCase): + encoding = 'cp949' + mapfileurl = 'http://www.pythontest.net/unicode/CP949.TXT' + + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: cp949 + def test_mapping_file(self): + return super().test_mapping_file() + + +class TestEUCKRMap(multibytecodec_support.TestBase_Mapping, + unittest.TestCase): + encoding = 'euc_kr' + mapfileurl = 'http://www.pythontest.net/unicode/EUC-KR.TXT' + + # A4D4 HANGUL FILLER indicates the begin of 8-bytes make-up sequence. + pass_enctest = [(b'\xa4\xd4', '\u3164')] + pass_dectest = [(b'\xa4\xd4', '\u3164')] + + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: euc_kr + def test_mapping_file(self): + return super().test_mapping_file() + + +class TestJOHABMap(multibytecodec_support.TestBase_Mapping, + unittest.TestCase): + encoding = 'johab' + mapfileurl = 'http://www.pythontest.net/unicode/JOHAB.TXT' + # KS X 1001 standard assigned 0x5c as WON SIGN. + # But the early 90s is the only era that used johab widely, + # most software implements it as REVERSE SOLIDUS. + # So, we ignore the standard here. + pass_enctest = [(b'\\', '\u20a9')] + pass_dectest = [(b'\\', '\u20a9')] + + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: johab + def test_mapping_file(self): + return super().test_mapping_file() + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_codecmaps_tw.py b/Lib/test/test_codecmaps_tw.py new file mode 100644 index 00000000000..4a1359ce7be --- /dev/null +++ b/Lib/test/test_codecmaps_tw.py @@ -0,0 +1,39 @@ +# +# test_codecmaps_tw.py +# Codec mapping tests for ROC encodings +# + +from test import multibytecodec_support +import unittest + +class TestBIG5Map(multibytecodec_support.TestBase_Mapping, + unittest.TestCase): + encoding = 'big5' + mapfileurl = 'http://www.pythontest.net/unicode/BIG5.TXT' + + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: big5 + def test_mapping_file(self): + return super().test_mapping_file() + +class TestCP950Map(multibytecodec_support.TestBase_Mapping, + unittest.TestCase): + encoding = 'cp950' + mapfileurl = 'http://www.pythontest.net/unicode/CP950.TXT' + pass_enctest = [ + (b'\xa2\xcc', '\u5341'), + (b'\xa2\xce', '\u5345'), + ] + codectests = ( + (b"\xFFxy", "replace", "\ufffdxy"), + ) + + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: cp950 + def test_errorhandle(self): + return super().test_errorhandle() + + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: cp950 + def test_mapping_file(self): + return super().test_mapping_file() + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index 736022599ed..fefd062eacb 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -1,12 +1,15 @@ import codecs import contextlib import copy +import importlib import io import pickle +import os import sys import unittest import encodings from unittest import mock +import warnings from test import support from test.support import os_helper @@ -20,13 +23,12 @@ except ImportError: _testinternalcapi = None -try: - import ctypes -except ImportError: - ctypes = None - SIZEOF_WCHAR_T = -1 -else: - SIZEOF_WCHAR_T = ctypes.sizeof(ctypes.c_wchar) + +def codecs_open_no_warn(*args, **kwargs): + """Call codecs.open(*args, **kwargs) ignoring DeprecationWarning.""" + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + return codecs.open(*args, **kwargs) def coding_checker(self, coder): def check(input, expect): @@ -35,13 +37,13 @@ def check(input, expect): # On small versions of Windows like Windows IoT or Windows Nano Server not all codepages are present def is_code_page_present(cp): - from ctypes import POINTER, WINFUNCTYPE, WinDLL + from ctypes import POINTER, WINFUNCTYPE, WinDLL, Structure from ctypes.wintypes import BOOL, BYTE, WCHAR, UINT, DWORD MAX_LEADBYTES = 12 # 5 ranges, 2 bytes ea., 0 term. MAX_DEFAULTCHAR = 2 # single or double byte MAX_PATH = 260 - class CPINFOEXW(ctypes.Structure): + class CPINFOEXW(Structure): _fields_ = [("MaxCharSize", UINT), ("DefaultChar", BYTE*MAX_DEFAULTCHAR), ("LeadByte", BYTE*MAX_LEADBYTES), @@ -388,7 +390,6 @@ def test_bug1098990_b(self): ill_formed_sequence_replace = "\ufffd" - @unittest.expectedFailure # TODO: RUSTPYTHON def test_lone_surrogates(self): self.assertRaises(UnicodeEncodeError, "\ud800".encode, self.encoding) self.assertEqual("[\uDC80]".encode(self.encoding, "backslashreplace"), @@ -464,7 +465,6 @@ class UTF32Test(ReadTest, unittest.TestCase): b'\x00\x00\x00s\x00\x00\x00p\x00\x00\x00a\x00\x00\x00m' b'\x00\x00\x00s\x00\x00\x00p\x00\x00\x00a\x00\x00\x00m') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_only_one_bom(self): _,_,reader,writer = codecs.lookup(self.encoding) # encode some stream @@ -480,7 +480,6 @@ def test_only_one_bom(self): f = reader(s) self.assertEqual(f.read(), "spamspam") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_badbom(self): s = io.BytesIO(4*b"\xff") f = codecs.getreader(self.encoding)(s) @@ -490,7 +489,6 @@ def test_badbom(self): f = codecs.getreader(self.encoding)(s) self.assertRaises(UnicodeDecodeError, f.read) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_partial(self): self.check_partial( "\x00\xff\u0100\uffff\U00010000", @@ -522,26 +520,22 @@ def test_partial(self): ] ) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_handlers(self): self.assertEqual(('\ufffd', 1), codecs.utf_32_decode(b'\x01', 'replace', True)) self.assertEqual(('', 1), codecs.utf_32_decode(b'\x01', 'ignore', True)) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_errors(self): self.assertRaises(UnicodeDecodeError, codecs.utf_32_decode, b"\xff", "strict", True) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_decoder_state(self): self.check_state_handling_decode(self.encoding, "spamspam", self.spamle) self.check_state_handling_decode(self.encoding, "spamspam", self.spambe) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_issue8941(self): # Issue #8941: insufficient result allocation when decoding into # surrogate pairs on UCS-2 builds. @@ -552,40 +546,11 @@ def test_issue8941(self): self.assertEqual('\U00010000' * 1024, codecs.utf_32_decode(encoded_be)[0]) - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_bug1098990_a(self): - return super().test_bug1098990_a() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_bug1098990_b(self): - return super().test_bug1098990_b() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_bug1175396(self): - return super().test_bug1175396() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_incremental_surrogatepass(self): - return super().test_incremental_surrogatepass() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_mixed_readline_and_read(self): - return super().test_mixed_readline_and_read() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_readline(self): - return super().test_readline() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_readlinequeue(self): - return super().test_readlinequeue() - class UTF32LETest(ReadTest, unittest.TestCase): encoding = "utf-32-le" ill_formed_sequence = b"\x80\xdc\x00\x00" - @unittest.expectedFailure # TODO: RUSTPYTHON def test_partial(self): self.check_partial( "\x00\xff\u0100\uffff\U00010000", @@ -613,16 +578,13 @@ def test_partial(self): ] ) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_simple(self): self.assertEqual("\U00010203".encode(self.encoding), b"\x03\x02\x01\x00") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_errors(self): self.assertRaises(UnicodeDecodeError, codecs.utf_32_le_decode, b"\xff", "strict", True) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_issue8941(self): # Issue #8941: insufficient result allocation when decoding into # surrogate pairs on UCS-2 builds. @@ -630,40 +592,11 @@ def test_issue8941(self): self.assertEqual('\U00010000' * 1024, codecs.utf_32_le_decode(encoded)[0]) - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_bug1098990_a(self): - return super().test_bug1098990_a() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_bug1098990_b(self): - return super().test_bug1098990_b() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_bug1175396(self): - return super().test_bug1175396() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_incremental_surrogatepass(self): - return super().test_incremental_surrogatepass() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_mixed_readline_and_read(self): - return super().test_mixed_readline_and_read() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_readline(self): - return super().test_readline() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_readlinequeue(self): - return super().test_readlinequeue() - class UTF32BETest(ReadTest, unittest.TestCase): encoding = "utf-32-be" ill_formed_sequence = b"\x00\x00\xdc\x80" - @unittest.expectedFailure # TODO: RUSTPYTHON def test_partial(self): self.check_partial( "\x00\xff\u0100\uffff\U00010000", @@ -691,16 +624,13 @@ def test_partial(self): ] ) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_simple(self): self.assertEqual("\U00010203".encode(self.encoding), b"\x00\x01\x02\x03") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_errors(self): self.assertRaises(UnicodeDecodeError, codecs.utf_32_be_decode, b"\xff", "strict", True) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_issue8941(self): # Issue #8941: insufficient result allocation when decoding into # surrogate pairs on UCS-2 builds. @@ -708,34 +638,6 @@ def test_issue8941(self): self.assertEqual('\U00010000' * 1024, codecs.utf_32_be_decode(encoded)[0]) - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_bug1098990_a(self): - return super().test_bug1098990_a() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_bug1098990_b(self): - return super().test_bug1098990_b() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_bug1175396(self): - return super().test_bug1175396() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_incremental_surrogatepass(self): - return super().test_incremental_surrogatepass() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_mixed_readline_and_read(self): - return super().test_mixed_readline_and_read() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_readline(self): - return super().test_readline() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_readlinequeue(self): - return super().test_readlinequeue() - class UTF16Test(ReadTest, unittest.TestCase): encoding = "utf-16" @@ -771,7 +673,6 @@ def test_badbom(self): f = codecs.getreader(self.encoding)(s) self.assertRaises(UnicodeDecodeError, f.read) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_partial(self): self.check_partial( "\x00\xff\u0100\uffff\U00010000", @@ -793,7 +694,6 @@ def test_partial(self): ] ) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_handlers(self): self.assertEqual(('\ufffd', 1), codecs.utf_16_decode(b'\x01', 'replace', True)) @@ -821,32 +721,27 @@ def test_bug691291(self): self.addCleanup(os_helper.unlink, os_helper.TESTFN) with open(os_helper.TESTFN, 'wb') as fp: fp.write(s) - with codecs.open(os_helper.TESTFN, 'r', + with codecs_open_no_warn(os_helper.TESTFN, 'r', encoding=self.encoding) as reader: self.assertEqual(reader.read(), s1) def test_invalid_modes(self): for mode in ('U', 'rU', 'r+U'): with self.assertRaises(ValueError) as cm: - codecs.open(os_helper.TESTFN, mode, encoding=self.encoding) + codecs_open_no_warn(os_helper.TESTFN, mode, encoding=self.encoding) self.assertIn('invalid mode', str(cm.exception)) for mode in ('rt', 'wt', 'at', 'r+t'): with self.assertRaises(ValueError) as cm: - codecs.open(os_helper.TESTFN, mode, encoding=self.encoding) + codecs_open_no_warn(os_helper.TESTFN, mode, encoding=self.encoding) self.assertIn("can't have text and binary mode at once", str(cm.exception)) - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_incremental_surrogatepass(self): - return super().test_incremental_surrogatepass() - class UTF16LETest(ReadTest, unittest.TestCase): encoding = "utf-16-le" ill_formed_sequence = b"\x80\xdc" - @unittest.expectedFailure # TODO: RUSTPYTHON def test_partial(self): self.check_partial( "\x00\xff\u0100\uffff\U00010000", @@ -866,7 +761,6 @@ def test_partial(self): ] ) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_errors(self): tests = [ (b'\xff', '\ufffd'), @@ -888,15 +782,10 @@ def test_nonbmp(self): self.assertEqual(b'\x00\xd8\x03\xde'.decode(self.encoding), "\U00010203") - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_incremental_surrogatepass(self): - return super().test_incremental_surrogatepass() - class UTF16BETest(ReadTest, unittest.TestCase): encoding = "utf-16-be" ill_formed_sequence = b"\xdc\x80" - @unittest.expectedFailure # TODO: RUSTPYTHON def test_partial(self): self.check_partial( "\x00\xff\u0100\uffff\U00010000", @@ -916,7 +805,6 @@ def test_partial(self): ] ) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_errors(self): tests = [ (b'\xff', '\ufffd'), @@ -938,10 +826,6 @@ def test_nonbmp(self): self.assertEqual(b'\xd8\x00\xde\x03'.decode(self.encoding), "\U00010203") - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_incremental_surrogatepass(self): - return super().test_incremental_surrogatepass() - class UTF8Test(ReadTest, unittest.TestCase): encoding = "utf-8" ill_formed_sequence = b"\xed\xb2\x80" @@ -987,7 +871,6 @@ def test_decode_error(self): self.assertEqual(data.decode(self.encoding, error_handler), expected) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_lone_surrogates(self): super().test_lone_surrogates() # not sure if this is making sense for @@ -1040,7 +923,6 @@ def test_incremental_errors(self): class UTF7Test(ReadTest, unittest.TestCase): encoding = "utf-7" - @unittest.expectedFailure # TODO: RUSTPYTHON def test_ascii(self): # Set D (directly encoded characters) set_d = ('ABCDEFGHIJKLMNOPQRSTUVWXYZ' @@ -1067,7 +949,6 @@ def test_ascii(self): b'+AAAAAQACAAMABAAFAAYABwAIAAsADAAOAA8AEAARABIAEwAU' b'ABUAFgAXABgAGQAaABsAHAAdAB4AHwBcAH4Afw-') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_partial(self): self.check_partial( 'a+-b\x00c\x80d\u0100e\U00010000f', @@ -1107,7 +988,6 @@ def test_partial(self): ] ) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_errors(self): tests = [ (b'\xffb', '\ufffdb'), @@ -1138,7 +1018,6 @@ def test_errors(self): raw, 'strict', True) self.assertEqual(raw.decode('utf-7', 'replace'), expected) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_nonbmp(self): self.assertEqual('\U000104A0'.encode(self.encoding), b'+2AHcoA-') self.assertEqual('\ud801\udca0'.encode(self.encoding), b'+2AHcoA-') @@ -1154,7 +1033,6 @@ def test_nonbmp(self): self.assertEqual(b'+IKwgrNgB3KA'.decode(self.encoding), '\u20ac\u20ac\U000104A0') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_lone_surrogates(self): tests = [ (b'a+2AE-b', 'a\ud801b'), @@ -1175,18 +1053,6 @@ def test_lone_surrogates(self): with self.subTest(raw=raw): self.assertEqual(raw.decode('utf-7', 'replace'), expected) - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_bug1175396(self): - return super().test_bug1175396() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_incremental_surrogatepass(self): - return super().test_incremental_surrogatepass() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_readline(self): - return super().test_readline() - class UTF16ExTest(unittest.TestCase): @@ -1310,7 +1176,6 @@ def test_raw(self): if b != b'\\': self.assertEqual(decode(b + b'0'), (b + b'0', 2)) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_escape(self): decode = codecs.escape_decode check = coding_checker(self, decode) @@ -1334,7 +1199,7 @@ def test_escape(self): check(br"[\x41]", b"[A]") check(br"[\x410]", b"[A0]") - @unittest.expectedFailure # TODO: RUSTPYTHON; DeprecationWarning not triggered + @unittest.expectedFailure # TODO: RUSTPYTHON; DeprecationWarning not triggered def test_warnings(self): decode = codecs.escape_decode check = coding_checker(self, decode) @@ -1342,32 +1207,31 @@ def test_warnings(self): b = bytes([i]) if b not in b'abfnrtvx': with self.assertWarnsRegex(DeprecationWarning, - r"invalid escape sequence '\\%c'" % i): + r'"\\%c" is an invalid escape sequence' % i): check(b"\\" + b, b"\\" + b) with self.assertWarnsRegex(DeprecationWarning, - r"invalid escape sequence '\\%c'" % (i-32)): + r'"\\%c" is an invalid escape sequence' % (i-32)): check(b"\\" + b.upper(), b"\\" + b.upper()) with self.assertWarnsRegex(DeprecationWarning, - r"invalid escape sequence '\\8'"): + r'"\\8" is an invalid escape sequence'): check(br"\8", b"\\8") with self.assertWarns(DeprecationWarning): check(br"\9", b"\\9") with self.assertWarnsRegex(DeprecationWarning, - r"invalid escape sequence '\\\xfa'") as cm: + r'"\\\xfa" is an invalid escape sequence') as cm: check(b"\\\xfa", b"\\\xfa") for i in range(0o400, 0o1000): with self.assertWarnsRegex(DeprecationWarning, - r"invalid octal escape sequence '\\%o'" % i): + r'"\\%o" is an invalid octal escape sequence' % i): check(rb'\%o' % i, bytes([i & 0o377])) with self.assertWarnsRegex(DeprecationWarning, - r"invalid escape sequence '\\z'"): + r'"\\z" is an invalid escape sequence'): self.assertEqual(decode(br'\x\z', 'ignore'), (b'\\z', 4)) with self.assertWarnsRegex(DeprecationWarning, - r"invalid octal escape sequence '\\501'"): + r'"\\501" is an invalid octal escape sequence'): self.assertEqual(decode(br'\x\501', 'ignore'), (b'A', 6)) - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: not raised by escape_decode def test_errors(self): decode = codecs.escape_decode self.assertRaises(ValueError, decode, br"\x") @@ -1999,9 +1863,9 @@ def test_all(self): def test_open(self): self.addCleanup(os_helper.unlink, os_helper.TESTFN) for mode in ('w', 'r', 'r+', 'w+', 'a', 'a+'): - with self.subTest(mode), \ - codecs.open(os_helper.TESTFN, mode, 'ascii') as file: - self.assertIsInstance(file, codecs.StreamReaderWriter) + with self.subTest(mode), self.assertWarns(DeprecationWarning): + with codecs.open(os_helper.TESTFN, mode, 'ascii') as file: + self.assertIsInstance(file, codecs.StreamReaderWriter) def test_undefined(self): self.assertRaises(UnicodeError, codecs.encode, 'abc', 'undefined') @@ -2018,7 +1882,7 @@ def test_file_closes_if_lookup_error_raised(self): mock_open = mock.mock_open() with mock.patch('builtins.open', mock_open) as file: with self.assertRaises(LookupError): - codecs.open(os_helper.TESTFN, 'wt', 'invalid-encoding') + codecs_open_no_warn(os_helper.TESTFN, 'wt', 'invalid-encoding') file().close.assert_called() @@ -2291,7 +2155,7 @@ def test_basic(self): class BasicUnicodeTest(unittest.TestCase, MixInCheckStateHandling): - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: big5 def test_basics(self): s = "abc123" # all codecs should be able to encode these for encoding in all_unicode_encodings: @@ -2411,7 +2275,7 @@ def test_basics_capi(self): self.assertEqual(decodedresult, s, "encoding=%r" % encoding) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: big5 def test_seek(self): # all codecs should be able to encode these s = "%s\n%s\n" % (100*"abc123", 100*"def456") @@ -2427,7 +2291,7 @@ def test_seek(self): data = reader.read() self.assertEqual(s, data) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: big5 def test_bad_decode_args(self): for encoding in all_unicode_encodings: decoder = codecs.getdecoder(encoding) @@ -2435,7 +2299,7 @@ def test_bad_decode_args(self): if encoding not in ("idna", "punycode"): self.assertRaises(TypeError, decoder, 42) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: big5 def test_bad_encode_args(self): for encoding in all_unicode_encodings: encoder = codecs.getencoder(encoding) @@ -2447,7 +2311,7 @@ def test_encoding_map_type_initialized(self): table_type = type(cp1140.encoding_table) self.assertEqual(table_type, table_type) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: big5 def test_decoder_state(self): # Check that getstate() and setstate() handle the state properly u = "abc123" @@ -2458,7 +2322,6 @@ def test_decoder_state(self): class CharmapTest(unittest.TestCase): - @unittest.expectedFailure # TODO: RUSTPYTHON def test_decode_with_string_map(self): self.assertEqual( codecs.charmap_decode(b"\x00\x01\x02", "strict", "abc"), @@ -2514,7 +2377,6 @@ def test_decode_with_string_map(self): ("", len(allbytes)) ) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_decode_with_int2str_map(self): self.assertEqual( codecs.charmap_decode(b"\x00\x01\x02", "strict", @@ -2631,7 +2493,6 @@ def test_decode_with_int2str_map(self): b"\x00\x01\x02", "strict", {0: "A", 1: 'Bb', 2: 999999999} ) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_decode_with_int2int_map(self): a = ord('a') b = ord('b') @@ -2724,7 +2585,7 @@ def test_streamreaderwriter(self): class TypesTest(unittest.TestCase): - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'codecs' has no attribute 'utf_32_ex_decode'. Did you mean: 'utf_16_ex_decode'? def test_decode_unicode(self): # Most decoders don't accept unicode input decoders = [ @@ -2826,7 +2687,7 @@ def test_escape_decode(self): check(br"\u20ac", "\u20ac") check(br"\U0001d120", "\U0001d120") - @unittest.expectedFailure # TODO: RUSTPYTHON; DeprecationWarning not triggered + @unittest.expectedFailure # TODO: RUSTPYTHON; DeprecationWarning not triggered def test_decode_warnings(self): decode = codecs.unicode_escape_decode check = coding_checker(self, decode) @@ -2834,30 +2695,30 @@ def test_decode_warnings(self): b = bytes([i]) if b not in b'abfnrtuvx': with self.assertWarnsRegex(DeprecationWarning, - r"invalid escape sequence '\\%c'" % i): + r'"\\%c" is an invalid escape sequence' % i): check(b"\\" + b, "\\" + chr(i)) if b.upper() not in b'UN': with self.assertWarnsRegex(DeprecationWarning, - r"invalid escape sequence '\\%c'" % (i-32)): + r'"\\%c" is an invalid escape sequence' % (i-32)): check(b"\\" + b.upper(), "\\" + chr(i-32)) with self.assertWarnsRegex(DeprecationWarning, - r"invalid escape sequence '\\8'"): + r'"\\8" is an invalid escape sequence'): check(br"\8", "\\8") with self.assertWarns(DeprecationWarning): check(br"\9", "\\9") with self.assertWarnsRegex(DeprecationWarning, - r"invalid escape sequence '\\\xfa'") as cm: + r'"\\\xfa" is an invalid escape sequence') as cm: check(b"\\\xfa", "\\\xfa") for i in range(0o400, 0o1000): with self.assertWarnsRegex(DeprecationWarning, - r"invalid octal escape sequence '\\%o'" % i): + r'"\\%o" is an invalid octal escape sequence' % i): check(rb'\%o' % i, chr(i)) with self.assertWarnsRegex(DeprecationWarning, - r"invalid escape sequence '\\z'"): + r'"\\z" is an invalid escape sequence'): self.assertEqual(decode(br'\x\z', 'ignore'), ('\\z', 4)) with self.assertWarnsRegex(DeprecationWarning, - r"invalid octal escape sequence '\\501'"): + r'"\\501" is an invalid octal escape sequence'): self.assertEqual(decode(br'\x\501', 'ignore'), ('\u0141', 6)) def test_decode_errors(self): @@ -2876,7 +2737,6 @@ def test_decode_errors(self): self.assertEqual(decode(br"\U00110000", "ignore"), ("", 10)) self.assertEqual(decode(br"\U00110000", "replace"), ("\ufffd", 10)) - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: '\x00\t\n\r\\' != '\x00\t\n\r' def test_partial(self): self.check_partial( "\x00\t\n\r\\\xff\uffff\U00010000", @@ -2916,14 +2776,10 @@ def test_partial(self): ] ) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; IndexError: index out of range def test_incremental_surrogatepass(self): return super().test_incremental_surrogatepass() - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_readline(self): - return super().test_readline() - class RawUnicodeEscapeTest(ReadTest, unittest.TestCase): encoding = "raw-unicode-escape" @@ -2977,7 +2833,6 @@ def test_decode_errors(self): self.assertEqual(decode(br"\U00110000", "ignore"), ("", 10)) self.assertEqual(decode(br"\U00110000", "replace"), ("\ufffd", 10)) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_partial(self): self.check_partial( "\x00\t\n\r\\\xff\uffff\U00010000", @@ -3007,14 +2862,6 @@ def test_partial(self): ] ) - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_incremental_surrogatepass(self): - return super().test_incremental_surrogatepass() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_readline(self): - return super().test_readline() - class EscapeEncodeTest(unittest.TestCase): @@ -3057,7 +2904,6 @@ def test_ascii(self): self.assertEqual("foo\udc80bar".encode("ascii", "surrogateescape"), b"foo\x80bar") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_charmap(self): # bad byte: \xa5 is unmapped in iso-8859-3 self.assertEqual(b"foo\xa5bar".decode("iso-8859-3", "surrogateescape"), @@ -3072,7 +2918,6 @@ def test_latin1(self): class BomTest(unittest.TestCase): - @unittest.expectedFailure # TODO: RUSTPYTHON def test_seek0(self): data = "1234567890" tests = ("utf-16", @@ -3084,7 +2929,7 @@ def test_seek0(self): self.addCleanup(os_helper.unlink, os_helper.TESTFN) for encoding in tests: # Check if the BOM is written only once - with codecs.open(os_helper.TESTFN, 'w+', encoding=encoding) as f: + with codecs_open_no_warn(os_helper.TESTFN, 'w+', encoding=encoding) as f: f.write(data) f.write(data) f.seek(0) @@ -3093,7 +2938,7 @@ def test_seek0(self): self.assertEqual(f.read(), data * 2) # Check that the BOM is written after a seek(0) - with codecs.open(os_helper.TESTFN, 'w+', encoding=encoding) as f: + with codecs_open_no_warn(os_helper.TESTFN, 'w+', encoding=encoding) as f: f.write(data[0]) self.assertNotEqual(f.tell(), 0) f.seek(0) @@ -3102,7 +2947,7 @@ def test_seek0(self): self.assertEqual(f.read(), data) # (StreamWriter) Check that the BOM is written after a seek(0) - with codecs.open(os_helper.TESTFN, 'w+', encoding=encoding) as f: + with codecs_open_no_warn(os_helper.TESTFN, 'w+', encoding=encoding) as f: f.writer.write(data[0]) self.assertNotEqual(f.writer.tell(), 0) f.writer.seek(0) @@ -3112,7 +2957,7 @@ def test_seek0(self): # Check that the BOM is not written after a seek() at a position # different than the start - with codecs.open(os_helper.TESTFN, 'w+', encoding=encoding) as f: + with codecs_open_no_warn(os_helper.TESTFN, 'w+', encoding=encoding) as f: f.write(data) f.seek(f.tell()) f.write(data) @@ -3121,7 +2966,7 @@ def test_seek0(self): # (StreamWriter) Check that the BOM is not written after a seek() # at a position different than the start - with codecs.open(os_helper.TESTFN, 'w+', encoding=encoding) as f: + with codecs_open_no_warn(os_helper.TESTFN, 'w+', encoding=encoding) as f: f.writer.write(data) f.writer.seek(f.writer.tell()) f.writer.write(data) @@ -3152,7 +2997,7 @@ def test_seek0(self): bytes_transform_encodings.append("zlib_codec") transform_aliases["zlib_codec"] = ["zip", "zlib"] try: - import bz2 + import bz2 # noqa: F401 except ImportError: pass else: @@ -3251,7 +3096,6 @@ def test_binary_to_text_denylists_text_transforms(self): bad_input.decode("rot_13") self.assertIsNone(failure.exception.__cause__) - @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipUnless(zlib, "Requires zlib support") def test_custom_zlib_error_is_noted(self): # Check zlib codec gives a good error for malformed input @@ -3260,7 +3104,6 @@ def test_custom_zlib_error_is_noted(self): codecs.decode(b"hello", "zlib_codec") self.assertEqual(msg, failure.exception.__notes__[0]) - @unittest.expectedFailure # TODO: RUSTPYTHON; - AttributeError: 'Error' object has no attribute '__notes__' def test_custom_hex_error_is_noted(self): # Check hex codec gives a good error for malformed input import binascii @@ -3278,6 +3121,13 @@ def test_aliases(self): info = codecs.lookup(alias) self.assertEqual(info.name, expected_name) + def test_alias_modules_exist(self): + encodings_dir = os.path.dirname(encodings.__file__) + for value in encodings.aliases.aliases.values(): + codec_mod = f"encodings.{value}" + self.assertIsNotNone(importlib.util.find_spec(codec_mod), + f"Codec module not found: {codec_mod}") + def test_quopri_stateless(self): # Should encode with quotetabs=True encoded = codecs.encode(b"space tab\teol \n", "quopri-codec") @@ -3341,7 +3191,6 @@ def raise_obj(self, *args, **kwds): # Helper to dynamically change the object raised by a test codec raise self.obj_to_raise - @unittest.expectedFailure # TODO: RUSTPYTHON def check_note(self, obj_to_raise, msg, exc_type=RuntimeError): self.obj_to_raise = obj_to_raise self.set_codec(self.raise_obj, self.raise_obj) @@ -3354,55 +3203,46 @@ def check_note(self, obj_to_raise, msg, exc_type=RuntimeError): with self.assertNoted("decoding", exc_type, msg): codecs.decode(b"bytes input", self.codec_name) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_raise_by_type(self): self.check_note(RuntimeError, "") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_raise_by_value(self): msg = "This should be noted" self.check_note(RuntimeError(msg), msg) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_raise_grandchild_subclass_exact_size(self): msg = "This should be noted" class MyRuntimeError(RuntimeError): __slots__ = () self.check_note(MyRuntimeError(msg), msg, MyRuntimeError) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_raise_subclass_with_weakref_support(self): msg = "This should be noted" class MyRuntimeError(RuntimeError): pass self.check_note(MyRuntimeError(msg), msg, MyRuntimeError) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_init_override(self): class CustomInit(RuntimeError): def __init__(self): pass self.check_note(CustomInit, "") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_new_override(self): class CustomNew(RuntimeError): def __new__(cls): return super().__new__(cls) self.check_note(CustomNew, "") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_instance_attribute(self): msg = "This should be noted" exc = RuntimeError(msg) exc.attr = 1 self.check_note(exc, "^{}$".format(msg)) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_non_str_arg(self): self.check_note(RuntimeError(1), "1") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_multiple_args(self): msg_re = r"^\('a', 'b', 'c'\)$" self.check_note(RuntimeError('a', 'b', 'c'), msg_re) @@ -3419,7 +3259,6 @@ def test_codec_lookup_failure(self): with self.assertRaisesRegex(LookupError, msg): codecs.decode(b"bytes input", self.codec_name) - @unittest.expectedFailure # TODO: RUSTPYTHON; def test_unflagged_non_text_codec_handling(self): # The stdlib non-text codecs are now marked so they're # pre-emptively skipped by the text model related methods @@ -3455,14 +3294,12 @@ def decode_to_bytes(*args, **kwds): class CodePageTest(unittest.TestCase): CP_UTF8 = 65001 - @unittest.expectedFailure # TODO: RUSTPYTHON def test_invalid_code_page(self): self.assertRaises(ValueError, codecs.code_page_encode, -1, 'a') self.assertRaises(ValueError, codecs.code_page_decode, -1, b'a') self.assertRaises(OSError, codecs.code_page_encode, 123, 'a') self.assertRaises(OSError, codecs.code_page_decode, 123, b'a') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_code_page_name(self): self.assertRaisesRegex(UnicodeEncodeError, 'cp932', codecs.code_page_encode, 932, '\xff') @@ -3472,7 +3309,11 @@ def test_code_page_name(self): codecs.code_page_decode, self.CP_UTF8, b'\xff', 'strict', True) def check_decode(self, cp, tests): - for raw, errors, expected in tests: + for raw, errors, expected, *rest in tests: + if rest: + altexpected, = rest + else: + altexpected = expected if expected is not None: try: decoded = codecs.code_page_decode(cp, raw, errors, True) @@ -3489,8 +3330,21 @@ def check_decode(self, cp, tests): self.assertRaises(UnicodeDecodeError, codecs.code_page_decode, cp, raw, errors, True) + if altexpected is not None: + decoded = raw.decode(f'cp{cp}', errors) + self.assertEqual(decoded, altexpected, + '%a.decode("cp%s", %r)=%a != %a' + % (raw, cp, errors, decoded, altexpected)) + else: + self.assertRaises(UnicodeDecodeError, + raw.decode, f'cp{cp}', errors) + def check_encode(self, cp, tests): - for text, errors, expected in tests: + for text, errors, expected, *rest in tests: + if rest: + altexpected, = rest + else: + altexpected = expected if expected is not None: try: encoded = codecs.code_page_encode(cp, text, errors) @@ -3501,19 +3355,27 @@ def check_encode(self, cp, tests): '%a.encode("cp%s", %r)=%a != %a' % (text, cp, errors, encoded[0], expected)) self.assertEqual(encoded[1], len(text)) + + encoded = text.encode(f'cp{cp}', errors) + self.assertEqual(encoded, altexpected, + '%a.encode("cp%s", %r)=%a != %a' + % (text, cp, errors, encoded, altexpected)) else: self.assertRaises(UnicodeEncodeError, codecs.code_page_encode, cp, text, errors) + self.assertRaises(UnicodeEncodeError, + text.encode, f'cp{cp}', errors) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_cp932(self): self.check_encode(932, ( ('abc', 'strict', b'abc'), ('\uff44\u9a3e', 'strict', b'\x82\x84\xe9\x80'), + ('\uf8f3', 'strict', b'\xff'), # test error handlers ('\xff', 'strict', None), ('[\xff]', 'ignore', b'[]'), - ('[\xff]', 'replace', b'[y]'), + ('[\xff]', 'replace', b'[y]', b'[?]'), ('[\u20ac]', 'replace', b'[?]'), ('[\xff]', 'backslashreplace', b'[\\xff]'), ('[\xff]', 'namereplace', @@ -3527,19 +3389,18 @@ def test_cp932(self): (b'abc', 'strict', 'abc'), (b'\x82\x84\xe9\x80', 'strict', '\uff44\u9a3e'), # invalid bytes - (b'[\xff]', 'strict', None), - (b'[\xff]', 'ignore', '[]'), - (b'[\xff]', 'replace', '[\ufffd]'), - (b'[\xff]', 'backslashreplace', '[\\xff]'), - (b'[\xff]', 'surrogateescape', '[\udcff]'), - (b'[\xff]', 'surrogatepass', None), + (b'[\xff]', 'strict', None, '[\uf8f3]'), + (b'[\xff]', 'ignore', '[]', '[\uf8f3]'), + (b'[\xff]', 'replace', '[\ufffd]', '[\uf8f3]'), + (b'[\xff]', 'backslashreplace', '[\\xff]', '[\uf8f3]'), + (b'[\xff]', 'surrogateescape', '[\udcff]', '[\uf8f3]'), + (b'[\xff]', 'surrogatepass', None, '[\uf8f3]'), (b'\x81\x00abc', 'strict', None), (b'\x81\x00abc', 'ignore', '\x00abc'), (b'\x81\x00abc', 'replace', '\ufffd\x00abc'), (b'\x81\x00abc', 'backslashreplace', '\\x81\x00abc'), )) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_cp1252(self): self.check_encode(1252, ( ('abc', 'strict', b'abc'), @@ -3548,7 +3409,7 @@ def test_cp1252(self): # test error handlers ('\u0141', 'strict', None), ('\u0141', 'ignore', b''), - ('\u0141', 'replace', b'L'), + ('\u0141', 'replace', b'L', b'?'), ('\udc98', 'surrogateescape', b'\x98'), ('\udc98', 'surrogatepass', None), )) @@ -3558,7 +3419,60 @@ def test_cp1252(self): (b'\xff', 'strict', '\xff'), )) - @unittest.expectedFailure # TODO: RUSTPYTHON + def test_cp708(self): + self.check_encode(708, ( + ('abc2%', 'strict', b'abc2%'), + ('\u060c\u0621\u064a', 'strict', b'\xac\xc1\xea'), + ('\u2562\xe7\xa0', 'strict', b'\x86\x87\xff'), + ('\x9a\x9f', 'strict', b'\x9a\x9f'), + ('\u256b', 'strict', b'\xc0'), + # test error handlers + ('[\u0662]', 'strict', None), + ('[\u0662]', 'ignore', b'[]'), + ('[\u0662]', 'replace', b'[?]'), + ('\udca0', 'surrogateescape', b'\xa0'), + ('\udca0', 'surrogatepass', None), + )) + self.check_decode(708, ( + (b'abc2%', 'strict', 'abc2%'), + (b'\xac\xc1\xea', 'strict', '\u060c\u0621\u064a'), + (b'\x86\x87\xff', 'strict', '\u2562\xe7\xa0'), + (b'\x9a\x9f', 'strict', '\x9a\x9f'), + (b'\xc0', 'strict', '\u256b'), + # test error handlers + (b'\xa0', 'strict', None), + (b'[\xa0]', 'ignore', '[]'), + (b'[\xa0]', 'replace', '[\ufffd]'), + (b'[\xa0]', 'backslashreplace', '[\\xa0]'), + (b'[\xa0]', 'surrogateescape', '[\udca0]'), + (b'[\xa0]', 'surrogatepass', None), + )) + + def test_cp20106(self): + self.check_encode(20106, ( + ('abc', 'strict', b'abc'), + ('\xa7\xc4\xdf', 'strict', b'@[~'), + # test error handlers + ('@', 'strict', None), + ('@', 'ignore', b''), + ('@', 'replace', b'?'), + ('\udcbf', 'surrogateescape', b'\xbf'), + ('\udcbf', 'surrogatepass', None), + )) + self.check_decode(20106, ( + (b'abc', 'strict', 'abc'), + (b'@[~', 'strict', '\xa7\xc4\xdf'), + (b'\xe1\xfe', 'strict', 'a\xdf'), + # test error handlers + (b'(\xbf)', 'strict', None), + (b'(\xbf)', 'ignore', '()'), + (b'(\xbf)', 'replace', '(\ufffd)'), + (b'(\xbf)', 'backslashreplace', '(\\xbf)'), + (b'(\xbf)', 'surrogateescape', '(\udcbf)'), + (b'(\xbf)', 'surrogatepass', None), + )) + + @unittest.expectedFailure # TODO: RUSTPYTHON; # TODO: RUSTPYTHON def test_cp_utf7(self): cp = 65000 self.check_encode(cp, ( @@ -3579,7 +3493,6 @@ def test_cp_utf7(self): (b'[\xff]', 'strict', '[\xff]'), )) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_multibyte_encoding(self): self.check_decode(932, ( (b'\x84\xe9\x80', 'ignore', '\u9a3e'), @@ -3594,7 +3507,6 @@ def test_multibyte_encoding(self): ('[\U0010ffff\uDC80]', 'replace', b'[\xf4\x8f\xbf\xbf?]'), )) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_code_page_decode_flags(self): # Issue #36312: For some code pages (e.g. UTF-7) flags for # MultiByteToWideChar() must be set to 0. @@ -3614,7 +3526,6 @@ def test_code_page_decode_flags(self): self.assertEqual(codecs.code_page_decode(42, b'abc'), ('\uf061\uf062\uf063', 3)) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_incremental(self): decoded = codecs.code_page_decode(932, b'\x82', 'strict', False) self.assertEqual(decoded, ('', 0)) @@ -3634,17 +3545,15 @@ def test_incremental(self): False) self.assertEqual(decoded, ('abc', 3)) - def test_mbcs_alias(self): - # Check that looking up our 'default' codepage will return - # mbcs when we don't have a more specific one available - code_page = 99_999 - name = f'cp{code_page}' - with mock.patch('_winapi.GetACP', return_value=code_page): - try: - codec = codecs.lookup(name) - self.assertEqual(codec.name, 'mbcs') - finally: - codecs.unregister(name) + def test_mbcs_code_page(self): + # Check that codec for the current Windows (ANSII) code page is + # always available. + try: + from _winapi import GetACP + except ImportError: + self.skipTest('requires _winapi.GetACP') + cp = GetACP() + codecs.lookup(f'cp{cp}') @support.bigmemtest(size=2**31, memuse=7, dry_run=False) def test_large_input(self, size): @@ -3908,7 +3817,7 @@ def check_decode_strings(self, errors): with self.assertRaises(RuntimeError) as cm: self.decode(encoded, errors) errmsg = str(cm.exception) - self.assertTrue(errmsg.startswith("decode error: "), errmsg) + self.assertStartsWith(errmsg, "decode error: ") else: decoded = self.decode(encoded, errors) self.assertEqual(decoded, expected) @@ -3977,7 +3886,6 @@ def test_rot13_func(self): class CodecNameNormalizationTest(unittest.TestCase): """Test codec name normalization""" - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: Tuples differ: (1, 2, 3, 4) != (None, None, None, None) def test_codecs_lookup(self): FOUND = (1, 2, 3, 4) NOT_FOUND = (None, None, None, None) diff --git a/Lib/test/test_codeop.py b/Lib/test/test_codeop.py index c62e3748e6a..12976122241 100644 --- a/Lib/test/test_codeop.py +++ b/Lib/test/test_codeop.py @@ -30,8 +30,6 @@ def assertInvalid(self, str, symbol='single', is_syntax=1): except OverflowError: self.assertTrue(not is_syntax) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_valid(self): av = self.assertValid @@ -94,8 +92,7 @@ def test_valid(self): av("def f():\n pass\n#foo\n") av("@a.b.c\ndef f():\n pass\n") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: at 0xc99532080 file "", line 1> != None def test_incomplete(self): ai = self.assertIncomplete @@ -282,13 +279,12 @@ def test_filename(self): self.assertNotEqual(compile_command("a = 1\n", "abc").co_filename, compile("a = 1\n", "def", 'single').co_filename) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 0 != 2 def test_warning(self): # Test that the warning is only returned once. with warnings_helper.check_warnings( ('"is" with \'str\' literal', SyntaxWarning), - ("invalid escape sequence", SyntaxWarning), + ('"\\\\e" is an invalid escape sequence', SyntaxWarning), ) as w: compile_command(r"'\e' is 0") self.assertEqual(len(w.warnings), 2) @@ -309,8 +305,7 @@ def test_incomplete_warning(self): self.assertIncomplete("'\\e' + (") self.assertEqual(w, []) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 0 != 1 def test_invalid_warning(self): with warnings.catch_warnings(record=True) as w: warnings.simplefilter('always') @@ -325,8 +320,6 @@ def assertSyntaxErrorMatches(self, code, message): with self.assertRaisesRegex(SyntaxError, message): compile_command(code, symbol='exec') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_syntax_errors(self): self.assertSyntaxErrorMatches( dedent("""\ diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index b68305dd7aa..7a57ba722e8 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -262,7 +262,6 @@ def __contains__(self, key): d = c.new_child(b=20, c=30) self.assertEqual(d.maps, [{'b': 20, 'c': 30}, {'a': 1, 'b': 2}]) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_union_operators(self): cm1 = ChainMap(dict(a=1, b=2), dict(c=3, d=4)) cm2 = ChainMap(dict(a=10, e=5), dict(b=20, d=4)) @@ -1957,7 +1956,7 @@ class X(ByteString): pass # No metaclass conflict class Z(ByteString, Awaitable): pass - @unittest.expectedFailure # TODO: RUSTPYTHON; Need to implement __buffer__ and __release_buffer__ (https://docs.python.org/3.13/reference/datamodel.html#emulating-buffer-types) + @unittest.expectedFailure # TODO: RUSTPYTHON; Need to implement __buffer__ and __release_buffer__ (https://docs.python.org/3.13/reference/datamodel.html#emulating-buffer-types) def test_Buffer(self): for sample in [bytes, bytearray, memoryview]: self.assertIsInstance(sample(b"x"), Buffer) @@ -2029,7 +2028,7 @@ def insert(self, index, value): self.assertEqual(len(mss), len(mss2)) self.assertEqual(list(mss), list(mss2)) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: TypeError not raised def test_illegal_patma_flags(self): with self.assertRaises(TypeError): class Both(Collection): @@ -2121,6 +2120,19 @@ def test_basics(self): self.assertEqual(c.setdefault('e', 5), 5) self.assertEqual(c['e'], 5) + def test_update_reentrant_add_clears_counter(self): + c = Counter() + key = object() + + class Evil(int): + def __add__(self, other): + c.clear() + return NotImplemented + + c[key] = Evil() + c.update([key]) + self.assertEqual(c[key], 1) + def test_init(self): self.assertEqual(list(Counter(self=42).items()), [('self', 42)]) self.assertEqual(list(Counter(iterable=42).items()), [('iterable', 42)]) diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 337366b4014..0495c58329c 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -1,5 +1,9 @@ +import contextlib import dis +import io +import itertools import math +import opcode import os import unittest import sys @@ -8,11 +12,18 @@ import tempfile import types import textwrap +import warnings +try: + import _testinternalcapi +except ImportError: + _testinternalcapi = None + from test import support -from test.support import script_helper, requires_debug_ranges +from test.support import (script_helper, requires_debug_ranges, run_code, + requires_specialization) +from test.support.bytecode_helper import instructions_with_positions from test.support.os_helper import FakePath - class TestSpecifics(unittest.TestCase): def compile_single(self, source): @@ -108,29 +119,30 @@ def __getitem__(self, key): exec('z = a', g, d) self.assertEqual(d['z'], 12) + @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") + @support.skip_emscripten_stack_overflow() def test_extended_arg(self): - # default: 1000 * 2.5 = 2500 repetitions - repeat = int(sys.getrecursionlimit() * 2.5) + repeat = 100 longexpr = 'x = x or ' + '-x' * repeat g = {} - code = ''' -def f(x): - %s - %s - %s - %s - %s - %s - %s - %s - %s - %s - # the expressions above have no effect, x == argument - while x: - x -= 1 - # EXTENDED_ARG/JUMP_ABSOLUTE here - return x -''' % ((longexpr,)*10) + code = textwrap.dedent(''' + def f(x): + %s + %s + %s + %s + %s + %s + %s + %s + %s + %s + # the expressions above have no effect, x == argument + while x: + x -= 1 + # EXTENDED_ARG/JUMP_ABSOLUTE here + return x + ''' % ((longexpr,)*10)) exec(code, g) self.assertEqual(g['f'](5), 0) @@ -146,10 +158,11 @@ def test_float_literals(self): def test_indentation(self): # testing compile() of indented block w/o trailing newline" - s = """ -if 1: - if 2: - pass""" + s = textwrap.dedent(""" + if 1: + if 2: + pass + """) compile(s, "", "exec") # This test is probably specific to CPython and may not generalize @@ -160,9 +173,8 @@ def test_leading_newlines(self): s256 = "".join(["\n"] * 256 + ["spam"]) co = compile(s256, 'fn', 'exec') self.assertEqual(co.co_firstlineno, 1) - lines = list(co.co_lines()) - self.assertEqual(lines[0][2], 0) - self.assertEqual(lines[1][2], 257) + lines = [line for _, _, line in co.co_lines()] + self.assertEqual(lines, [0, 257]) def test_literals_with_leading_zeroes(self): for arg in ["077787", "0xj", "0x.", "0e", "090000000000000", @@ -197,8 +209,7 @@ def test_literals_with_leading_zeroes(self): self.assertEqual(eval("0o777"), 511) self.assertEqual(eval("-0o0000010"), -8) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: SyntaxError not raised def test_int_literals_too_long(self): n = 3000 source = f"a = 1\nb = 2\nc = {'3'*n}\nd = 4" @@ -272,8 +283,7 @@ def test_none_assignment(self): self.assertRaises(SyntaxError, compile, stmt, 'tmp', 'single') self.assertRaises(SyntaxError, compile, stmt, 'tmp', 'exec') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: SyntaxError not raised by compile def test_import(self): succeed = [ 'import sys', @@ -334,8 +344,11 @@ def test_lambda_doc(self): l = lambda: "foo" self.assertIsNone(l.__doc__) - # TODO: RUSTPYTHON - @unittest.expectedFailure + def test_lambda_consts(self): + l = lambda: "this is the only const" + self.assertEqual(l.__code__.co_consts, ("this is the only const",)) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: SyntaxError not raised by compile def test_encoding(self): code = b'# -*- coding: badencoding -*-\npass\n' self.assertRaises(SyntaxError, compile, code, 'tmp', 'exec') @@ -440,16 +453,134 @@ class A: def f(): __mangled = 1 __not_mangled__ = 2 - import __mangled_mod - import __package__.module + import __mangled_mod # noqa: F401 + import __package__.module # noqa: F401 self.assertIn("_A__mangled", A.f.__code__.co_varnames) self.assertIn("__not_mangled__", A.f.__code__.co_varnames) self.assertIn("_A__mangled_mod", A.f.__code__.co_varnames) self.assertIn("__package__", A.f.__code__.co_varnames) - # TODO: RUSTPYTHON - @unittest.expectedFailure + def test_condition_expression_with_dead_blocks_compiles(self): + # See gh-113054 + compile('if (5 if 5 else T): 0', '', 'exec') + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_condition_expression_with_redundant_comparisons_compiles(self): + # See gh-113054, gh-114083 + exprs = [ + 'if 9<9<9and 9or 9:9', + 'if 9<9<9and 9or 9or 9:9', + 'if 9<9<9and 9or 9or 9or 9:9', + 'if 9<9<9and 9or 9or 9or 9or 9:9', + ] + for expr in exprs: + with self.subTest(expr=expr): + with self.assertWarns(SyntaxWarning): + compile(expr, '', 'exec') + + def test_dead_code_with_except_handler_compiles(self): + compile(textwrap.dedent(""" + if None: + with CM: + x = 1 + else: + x = 2 + """), '', 'exec') + + def test_try_except_in_while_with_chained_condition_compiles(self): + # see gh-124871 + compile(textwrap.dedent(""" + name_1, name_2, name_3 = 1, 2, 3 + while name_3 <= name_2 > name_1: + try: + raise + except: + pass + finally: + pass + """), '', 'exec') + + def test_compile_invalid_namedexpr(self): + # gh-109351 + m = ast.Module( + body=[ + ast.Expr( + value=ast.ListComp( + elt=ast.NamedExpr( + target=ast.Constant(value=1), + value=ast.Constant(value=3), + ), + generators=[ + ast.comprehension( + target=ast.Name(id="x", ctx=ast.Store()), + iter=ast.Name(id="y", ctx=ast.Load()), + ifs=[], + is_async=0, + ) + ], + ) + ) + ], + type_ignores=[], + ) + + with self.assertRaisesRegex(TypeError, "NamedExpr target must be a Name"): + compile(ast.fix_missing_locations(m), "", "exec") + + def test_compile_redundant_jumps_and_nops_after_moving_cold_blocks(self): + # See gh-120367 + code=textwrap.dedent(""" + try: + pass + except: + pass + else: + match name_2: + case b'': + pass + finally: + something + """) + + tree = ast.parse(code) + + # make all instruction locations the same to create redundancies + for node in ast.walk(tree): + if hasattr(node,"lineno"): + del node.lineno + del node.end_lineno + del node.col_offset + del node.end_col_offset + + compile(ast.fix_missing_locations(tree), "", "exec") + + def test_compile_redundant_jump_after_convert_pseudo_ops(self): + # See gh-120367 + code=textwrap.dedent(""" + if name_2: + pass + else: + try: + pass + except: + pass + ~name_5 + """) + + tree = ast.parse(code) + + # make all instruction locations the same to create redundancies + for node in ast.walk(tree): + if hasattr(node,"lineno"): + del node.lineno + del node.end_lineno + del node.col_offset + del node.end_col_offset + + compile(ast.fix_missing_locations(tree), "", "exec") + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: at 0xb77555080 file "1", line 1> != at 0xb77554f00 file "3", line 1> def test_compile_ast(self): fname = __file__ if fname.lower().endswith('pyc'): @@ -478,13 +609,33 @@ def test_compile_ast(self): self.assertRaises(TypeError, compile, co1, '', 'eval') # raise exception when node type is no start node - self.assertRaises(TypeError, compile, _ast.If(), '', 'exec') + self.assertRaises(TypeError, compile, _ast.If(test=_ast.Name(id='x', ctx=_ast.Load())), '', 'exec') # raise exception when node has invalid children ast = _ast.Module() - ast.body = [_ast.BoolOp()] + ast.body = [_ast.BoolOp(op=_ast.Or())] self.assertRaises(TypeError, compile, ast, '', 'exec') + def test_compile_invalid_typealias(self): + # gh-109341 + m = ast.Module( + body=[ + ast.TypeAlias( + name=ast.Subscript( + value=ast.Name(id="foo", ctx=ast.Load()), + slice=ast.Constant(value="x"), + ctx=ast.Store(), + ), + type_params=[], + value=ast.Name(id="Callable", ctx=ast.Load()), + ) + ], + type_ignores=[], + ) + + with self.assertRaisesRegex(TypeError, "TypeAlias with non-Name name"): + compile(ast.fix_missing_locations(m), "", "exec") + def test_dict_evaluation_order(self): i = 0 @@ -496,18 +647,32 @@ def f(): d = {f(): f(), f(): f()} self.assertEqual(d, {1: 2, 3: 4}) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: TypeError not raised def test_compile_filename(self): for filename in 'file.py', b'file.py': code = compile('pass', filename, 'exec') self.assertEqual(code.co_filename, 'file.py') for filename in bytearray(b'file.py'), memoryview(b'file.py'): - with self.assertWarns(DeprecationWarning): - code = compile('pass', filename, 'exec') - self.assertEqual(code.co_filename, 'file.py') + with self.assertRaises(TypeError): + compile('pass', filename, 'exec') self.assertRaises(TypeError, compile, 'pass', list(b'file.py'), 'exec') + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Expected type bool, not EvilBool + def test_compile_filename_refleak(self): + # Regression tests for reference leak in PyUnicode_FSDecoder. + # See https://github.com/python/cpython/issues/139748. + mortal_str = 'this is a mortal string' + # check error path when 'mode' AC conversion failed + self.assertRaises(TypeError, compile, b'', mortal_str, mode=1234) + # check error path when 'optimize' AC conversion failed + self.assertRaises(OverflowError, compile, b'', mortal_str, + 'exec', optimize=1 << 1000) + # check error path when 'dont_inherit' AC conversion failed + class EvilBool: + def __bool__(self): raise ValueError + self.assertRaises(ValueError, compile, b'', mortal_str, + 'exec', dont_inherit=EvilBool()) + @support.cpython_only def test_same_filename_used(self): s = """def f(): pass\ndef g(): pass""" @@ -533,8 +698,7 @@ def test_single_statement(self): self.compile_single("class T:\n pass") self.compile_single("c = '''\na=1\nb=2\nc=3\n'''") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: SyntaxError not raised by compile_single def test_bad_single_statement(self): self.assertInvalidSingle('1\n2') self.assertInvalidSingle('def f(): pass') @@ -546,8 +710,7 @@ def test_bad_single_statement(self): self.assertInvalidSingle('x = 5 # comment\nx = 6\n') self.assertInvalidSingle("c = '''\nd=1\n'''\na = 1\n\nb = 2\n") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: b'source code cannot contain null bytes' not found in b'OSError: stream did not contain valid UTF-8\n' def test_particularly_evil_undecodable(self): # Issue 24022 src = b'0000\x00\n00000000000\n\x00\n\x9e\n' @@ -556,10 +719,9 @@ def test_particularly_evil_undecodable(self): with open(fn, "wb") as fp: fp.write(src) res = script_helper.run_python_until_end(fn)[0] - self.assertIn(b"Non-UTF-8", res.err) + self.assertIn(b"source code cannot contain null bytes", res.err) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: b'source code cannot contain null bytes' not found in b'OSError: stream did not contain valid UTF-8\n' def test_yet_more_evil_still_undecodable(self): # Issue #25388 src = b"#\x00\n#\xfd\n" @@ -568,30 +730,25 @@ def test_yet_more_evil_still_undecodable(self): with open(fn, "wb") as fp: fp.write(src) res = script_helper.run_python_until_end(fn)[0] - self.assertIn(b"Non-UTF-8", res.err) + self.assertIn(b"source code cannot contain null bytes", res.err) @support.cpython_only + @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") + @support.skip_emscripten_stack_overflow() def test_compiler_recursion_limit(self): - # Expected limit is sys.getrecursionlimit() * the scaling factor - # in symtable.c (currently 3) - # We expect to fail *at* that limit, because we use up some of - # the stack depth limit in the test suite code - # So we check the expected limit and 75% of that - # XXX (ncoghlan): duplicating the scaling factor here is a little - # ugly. Perhaps it should be exposed somewhere... - fail_depth = sys.getrecursionlimit() * 3 - crash_depth = sys.getrecursionlimit() * 300 - success_depth = int(fail_depth * 0.75) + # Compiler frames are small + limit = 100 + # Android test devices have less memory. + crash_depth = limit * (1000 if sys.platform == "android" else 5000) + success_depth = limit def check_limit(prefix, repeated, mode="single"): expect_ok = prefix + repeated * success_depth compile(expect_ok, '', mode) - for depth in (fail_depth, crash_depth): - broken = prefix + repeated * depth - details = "Compiling ({!r} + {!r} * {})".format( - prefix, repeated, depth) - with self.assertRaises(RecursionError, msg=details): - compile(broken, '', mode) + broken = prefix + repeated * crash_depth + details = f"Compiling ({prefix!r} + {repeated!r} * {crash_depth})" + with self.assertRaises(RecursionError, msg=details): + compile(broken, '', mode) check_limit("a", "()") check_limit("a", ".b") @@ -601,14 +758,13 @@ def check_limit(prefix, repeated, mode="single"): # check_limit("a", " if a else a") # check_limit("if a: pass", "\nelif a: pass", mode="exec") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: "cannot contain null" does not match "invalid syntax (, line 1)" def test_null_terminated(self): # The source code is null-terminated internally, but bytes-like # objects are accepted, which could be not terminated. - with self.assertRaisesRegex(ValueError, "cannot contain null"): + with self.assertRaisesRegex(SyntaxError, "cannot contain null"): compile("123\x00", "", "eval") - with self.assertRaisesRegex(ValueError, "cannot contain null"): + with self.assertRaisesRegex(SyntaxError, "cannot contain null"): compile(memoryview(b"123\x00"), "", "eval") code = compile(memoryview(b"123\x00")[1:-1], "", "eval") self.assertEqual(eval(code), 23) @@ -649,7 +805,6 @@ def check_same_constant(const): self.assertEqual(repr(f1()), repr(const)) check_same_constant(None) - check_same_constant(0) check_same_constant(0.0) check_same_constant(b'abc') check_same_constant('abc') @@ -664,7 +819,7 @@ def check_same_constant(const): # Merge constants in tuple or frozenset f1, f2 = lambda: "not a name", lambda: ("not a name",) f3 = lambda x: x in {("not a name",)} - self.assertIs(f1.__code__.co_consts[1], + self.assertIs(f1.__code__.co_consts[0], f2.__code__.co_consts[1][0]) self.assertIs(next(iter(f3.__code__.co_consts[1])), f2.__code__.co_consts[1]) @@ -686,16 +841,62 @@ def test_merge_code_attrs(self): self.assertIs(f1.__code__.co_linetable, f2.__code__.co_linetable) + @support.cpython_only + def test_remove_unused_consts(self): + def f(): + "docstring" + if True: + return "used" + else: + return "unused" + + self.assertEqual(f.__code__.co_consts, + (f.__doc__, "used")) + + @support.cpython_only + def test_remove_unused_consts_no_docstring(self): + # the first item (None for no docstring in this case) is + # always retained. + def f(): + if True: + return "used" + else: + return "unused" + + self.assertEqual(f.__code__.co_consts, + (True, "used")) + + @support.cpython_only + def test_remove_unused_consts_extended_args(self): + N = 1000 + code = ["def f():\n"] + code.append("\ts = ''\n") + code.append("\tfor i in range(1):\n") + for i in range(N): + code.append(f"\t\tif True: s += 't{i}'\n") + code.append(f"\t\tif False: s += 'f{i}'\n") + code.append("\treturn s\n") + + code = "".join(code) + g = {} + eval(compile(code, "file.py", "exec"), g) + exec(code, g) + f = g['f'] + expected = tuple([''] + [f't{i}' for i in range(N)]) + self.assertEqual(f.__code__.co_consts, expected) + expected = "".join(expected[1:]) + self.assertEqual(expected, f()) + # Stripping unused constants is not a strict requirement for the # Python semantics, it's a more an implementation detail. @support.cpython_only - def test_strip_unused_consts(self): + def test_strip_unused_None(self): # Python 3.10rc1 appended None to co_consts when None is not used # at all. See bpo-45056. def f1(): "docstring" return 42 - self.assertEqual(f1.__code__.co_consts, ("docstring", 42)) + self.assertEqual(f1.__code__.co_consts, (f1.__doc__,)) # This is a regression test for a CPython specific peephole optimizer # implementation bug present in a few releases. It's assertion verifies @@ -715,8 +916,90 @@ def unused_code_at_end(): 'RETURN_VALUE', list(dis.get_instructions(unused_code_at_end))[-1].opname) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @support.cpython_only + def test_docstring(self): + src = textwrap.dedent(""" + def with_docstring(): + "docstring" + + def two_strings(): + "docstring" + "not docstring" + + def with_fstring(): + f"not docstring" + + def with_const_expression(): + "also" + " not docstring" + + def multiple_const_strings(): + "not docstring " * 3 + """) + + for opt in [0, 1, 2]: + with self.subTest(opt=opt): + code = compile(src, "", "exec", optimize=opt) + ns = {} + exec(code, ns) + + if opt < 2: + self.assertEqual(ns['with_docstring'].__doc__, "docstring") + self.assertEqual(ns['two_strings'].__doc__, "docstring") + else: + self.assertIsNone(ns['with_docstring'].__doc__) + self.assertIsNone(ns['two_strings'].__doc__) + self.assertIsNone(ns['with_fstring'].__doc__) + self.assertIsNone(ns['with_const_expression'].__doc__) + self.assertIsNone(ns['multiple_const_strings'].__doc__) + + @support.cpython_only + def test_docstring_interactive_mode(self): + srcs = [ + """def with_docstring(): + "docstring" + """, + """class with_docstring: + "docstring" + """, + ] + + for opt in [0, 1, 2]: + for src in srcs: + with self.subTest(opt=opt, src=src): + code = compile(textwrap.dedent(src), "", "single", optimize=opt) + ns = {} + exec(code, ns) + if opt < 2: + self.assertEqual(ns['with_docstring'].__doc__, "docstring") + else: + self.assertIsNone(ns['with_docstring'].__doc__) + + @support.cpython_only + def test_docstring_omitted(self): + # See gh-115347 + src = textwrap.dedent(""" + def f(): + "docstring1" + def h(): + "docstring2" + return 42 + + class C: + "docstring3" + pass + + return h + """) + for opt in [-1, 0, 1, 2]: + for mode in ["exec", "single"]: + with self.subTest(opt=opt, mode=mode): + code = compile(src, "", mode, optimize=opt) + output = io.StringIO() + with contextlib.redirect_stdout(output): + dis.dis(code) + self.assertNotIn('NOP', output.getvalue()) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: unable to find constant -0.0 in (0.0,) def test_dont_merge_constants(self): # Issue #25843: compile() must not merge constants which are equal # but have a different type. @@ -733,7 +1016,6 @@ def check_different_constants(const1, const2): self.assertEqual(repr(f1()), repr(const1)) self.assertEqual(repr(f2()), repr(const2)) - check_different_constants(0, 0.0) check_different_constants(+0.0, -0.0) check_different_constants((0,), (0.0,)) check_different_constants('a', b'a') @@ -761,10 +1043,13 @@ def test_path_like_objects(self): # An implicit test for PyUnicode_FSDecoder(). compile("42", FakePath("test_compile_pathlike"), "single") + # bpo-31113: Stack overflow when compile a long sequence of + # complex statements. + @support.requires_resource('cpu') def test_stack_overflow(self): - # bpo-31113: Stack overflow when compile a long sequence of - # complex statements. - compile("if a: b\n" * 200000, "", "exec") + # Android test devices have less memory. + size = 100_000 if sys.platform == "android" else 200_000 + compile("if a: b\n" * size, "", "exec") # Multiple users rely on the fact that CPython does not generate # bytecode for dead code blocks. See bpo-37500 for more context. @@ -796,12 +1081,10 @@ def unused_block_while_else(): for func in funcs: opcodes = list(dis.get_instructions(func)) self.assertLessEqual(len(opcodes), 4) - self.assertEqual('LOAD_CONST', opcodes[-2].opname) - self.assertEqual(None, opcodes[-2].argval) self.assertEqual('RETURN_VALUE', opcodes[-1].opname) + self.assertEqual(None, opcodes[-1].argval) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 3 != 8 def test_false_while_loop(self): def break_in_while(): while False: @@ -817,12 +1100,10 @@ def continue_in_while(): for func in funcs: opcodes = list(dis.get_instructions(func)) self.assertEqual(3, len(opcodes)) - self.assertEqual('LOAD_CONST', opcodes[1].opname) + self.assertEqual('RETURN_VALUE', opcodes[-1].opname) self.assertEqual(None, opcodes[1].argval) - self.assertEqual('RETURN_VALUE', opcodes[2].opname) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_consts_in_conditionals(self): def and_true(x): return True and x @@ -846,8 +1127,6 @@ def or_false(x): self.assertIn('LOAD_', opcodes[-2].opname) self.assertEqual('RETURN_VALUE', opcodes[-1].opname) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_imported_load_method(self): sources = [ """\ @@ -880,7 +1159,33 @@ def foo(x): instructions = [opcode.opname for opcode in opcodes] self.assertNotIn('LOAD_METHOD', instructions) self.assertIn('LOAD_ATTR', instructions) - self.assertIn('PRECALL', instructions) + self.assertIn('CALL', instructions) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 'LOAD_SMALL_INT' not found in ['RESUME', 'LOAD_CONST', 'RETURN_VALUE'] + def test_folding_type_param(self): + get_code_fn_cls = lambda x: x.co_consts[0].co_consts[2] + get_code_type_alias = lambda x: x.co_consts[0].co_consts[3] + snippets = [ + ("def foo[T = 40 + 5](): pass", get_code_fn_cls), + ("def foo[**P = 40 + 5](): pass", get_code_fn_cls), + ("def foo[*Ts = 40 + 5](): pass", get_code_fn_cls), + ("class foo[T = 40 + 5]: pass", get_code_fn_cls), + ("class foo[**P = 40 + 5]: pass", get_code_fn_cls), + ("class foo[*Ts = 40 + 5]: pass", get_code_fn_cls), + ("type foo[T = 40 + 5] = 1", get_code_type_alias), + ("type foo[**P = 40 + 5] = 1", get_code_type_alias), + ("type foo[*Ts = 40 + 5] = 1", get_code_type_alias), + ] + for snippet, get_code in snippets: + c = compile(snippet, "", "exec") + code = get_code(c) + opcodes = list(dis.get_instructions(code)) + instructions = [opcode.opname for opcode in opcodes] + args = [opcode.oparg for opcode in opcodes] + self.assertNotIn(40, args) + self.assertNotIn(5, args) + self.assertIn('LOAD_SMALL_INT', instructions) + self.assertIn(45, args) def test_lineno_procedure_call(self): def call(): @@ -890,8 +1195,7 @@ def call(): line1 = call.__code__.co_firstlineno + 1 assert line1 not in [line for (_, _, line) in call.__code__.co_lines()] - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_lineno_after_implicit_return(self): TRUE = True # Don't use constant True or False, as compiler will remove test @@ -935,10 +1239,12 @@ def no_code2(): for func in (no_code1, no_code2): with self.subTest(func=func): + if func is no_code1 and no_code1.__doc__ is None: + continue code = func.__code__ - lines = list(code.co_lines()) - start, end, line = lines[0] + [(start, end, line)] = code.co_lines() self.assertEqual(start, 0) + self.assertEqual(end, len(code.co_code)) self.assertEqual(line, code.co_firstlineno) def get_code_lines(self, code): @@ -950,8 +1256,7 @@ def get_code_lines(self, code): last_line = line return res - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_lineno_attribute(self): def load_attr(): return ( @@ -996,8 +1301,7 @@ def aug_store_attr(): code_lines = self.get_code_lines(func.__code__) self.assertEqual(lines, code_lines) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; + [0] def test_line_number_genexp(self): def return_genexp(): @@ -1006,14 +1310,12 @@ def return_genexp(): x in y) - genexp_lines = [0, 2, 0] + genexp_lines = [0, 4, 2, 0, 4] - genexp_code = return_genexp.__code__.co_consts[1] + genexp_code = return_genexp.__code__.co_consts[0] code_lines = self.get_code_lines(genexp_code) self.assertEqual(genexp_lines, code_lines) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_line_number_implicit_return_after_async_for(self): async def test(aseq): @@ -1024,6 +1326,73 @@ async def test(aseq): code_lines = self.get_code_lines(test.__code__) self.assertEqual(expected_lines, code_lines) + def check_line_numbers(self, code, opnames=None): + # Check that all instructions whose op matches opnames + # have a line number. opnames can be a single name, or + # a sequence of names. If it is None, match all ops. + + if isinstance(opnames, str): + opnames = (opnames, ) + for inst in dis.Bytecode(code): + if opnames and inst.opname in opnames: + self.assertIsNotNone(inst.positions.lineno) + + def test_line_number_synthetic_jump_multiple_predecessors(self): + def f(): + for x in it: + try: + if C1: + yield 2 + except OSError: + pass + + self.check_line_numbers(f.__code__, 'JUMP_BACKWARD') + + def test_line_number_synthetic_jump_multiple_predecessors_nested(self): + def f(): + for x in it: + try: + X = 3 + except OSError: + try: + if C3: + X = 4 + except OSError: + pass + return 42 + + self.check_line_numbers(f.__code__, 'JUMP_BACKWARD') + + def test_line_number_synthetic_jump_multiple_predecessors_more_nested(self): + def f(): + for x in it: + try: + X = 3 + except OSError: + try: + if C3: + if C4: + X = 4 + except OSError: + try: + if C3: + if C4: + X = 5 + except OSError: + pass + return 42 + + self.check_line_numbers(f.__code__, 'JUMP_BACKWARD') + + def test_lineno_of_backward_jump_conditional_in_loop(self): + # Issue gh-107901 + def f(): + for i in x: + if y: + pass + + self.check_line_numbers(f.__code__, 'JUMP_BACKWARD') + def test_big_dict_literal(self): # The compiler has a flushing point in "compiler_dict" that calls compiles # a portion of the dictionary literal when the loop that iterates over the items @@ -1073,14 +1442,109 @@ def while_not_chained(a, b, c): for instr in dis.Bytecode(while_not_chained): self.assertNotEqual(instr.opname, "EXTENDED_ARG") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @support.cpython_only + def test_uses_slice_instructions(self): + + def check_op_count(func, op, expected): + actual = 0 + for instr in dis.Bytecode(func): + if instr.opname == op: + actual += 1 + self.assertEqual(actual, expected) + + def check_consts(func, typ, expected): + expected = set([repr(x) for x in expected]) + all_consts = set() + consts = func.__code__.co_consts + for instr in dis.Bytecode(func): + if instr.opname == "LOAD_CONST" and isinstance(consts[instr.oparg], typ): + all_consts.add(repr(consts[instr.oparg])) + self.assertEqual(all_consts, expected) + + def load(): + return x[a:b] + x [a:] + x[:b] + x[:] + + check_op_count(load, "BINARY_SLICE", 3) + check_op_count(load, "BUILD_SLICE", 0) + check_consts(load, slice, [slice(None, None, None)]) + check_op_count(load, "BINARY_OP", 4) + + def store(): + x[a:b] = y + x [a:] = y + x[:b] = y + x[:] = y + + check_op_count(store, "STORE_SLICE", 3) + check_op_count(store, "BUILD_SLICE", 0) + check_op_count(store, "STORE_SUBSCR", 1) + check_consts(store, slice, [slice(None, None, None)]) + + def long_slice(): + return x[a:b:c] + + check_op_count(long_slice, "BUILD_SLICE", 1) + check_op_count(long_slice, "BINARY_SLICE", 0) + check_consts(long_slice, slice, []) + check_op_count(long_slice, "BINARY_OP", 1) + + def aug(): + x[a:b] += y + + check_op_count(aug, "BINARY_SLICE", 1) + check_op_count(aug, "STORE_SLICE", 1) + check_op_count(aug, "BUILD_SLICE", 0) + check_op_count(aug, "BINARY_OP", 1) + check_op_count(aug, "STORE_SUBSCR", 0) + check_consts(aug, slice, []) + + def aug_const(): + x[1:2] += y + + check_op_count(aug_const, "BINARY_SLICE", 0) + check_op_count(aug_const, "STORE_SLICE", 0) + check_op_count(aug_const, "BINARY_OP", 2) + check_op_count(aug_const, "STORE_SUBSCR", 1) + check_consts(aug_const, slice, [slice(1, 2)]) + + def compound_const_slice(): + x[1:2:3, 4:5:6] = y + + check_op_count(compound_const_slice, "BINARY_SLICE", 0) + check_op_count(compound_const_slice, "BUILD_SLICE", 0) + check_op_count(compound_const_slice, "STORE_SLICE", 0) + check_op_count(compound_const_slice, "STORE_SUBSCR", 1) + check_consts(compound_const_slice, slice, []) + check_consts(compound_const_slice, tuple, [(slice(1, 2, 3), slice(4, 5, 6))]) + + def mutable_slice(): + x[[]:] = y + + check_consts(mutable_slice, slice, {}) + + def different_but_equal(): + x[:0] = y + x[:0.0] = y + x[:False] = y + x[:None] = y + + check_consts( + different_but_equal, + slice, + [ + slice(None, 0, None), + slice(None, 0.0, None), + slice(None, False, None), + slice(None, None, None) + ] + ) + def test_compare_positions(self): - for opname, op in [ - ("COMPARE_OP", "<"), - ("COMPARE_OP", "<="), - ("COMPARE_OP", ">"), - ("COMPARE_OP", ">="), + for opname_prefix, op in [ + ("COMPARE_", "<"), + ("COMPARE_", "<="), + ("COMPARE_", ">"), + ("COMPARE_", ">="), ("CONTAINS_OP", "in"), ("CONTAINS_OP", "not in"), ("IS_OP", "is"), @@ -1095,11 +1559,313 @@ def test_compare_positions(self): actual_positions = [ instruction.positions for instruction in dis.get_instructions(code) - if instruction.opname == opname + if instruction.opname.startswith(opname_prefix) ] with self.subTest(source): self.assertEqual(actual_positions, expected_positions) + def test_if_expression_expression_empty_block(self): + # See regression in gh-99708 + exprs = [ + "assert (False if 1 else True)", + "def f():\n\tif not (False if 1 else True): raise AssertionError", + "def f():\n\tif not (False if 1 else True): return 12", + ] + for expr in exprs: + with self.subTest(expr=expr): + compile(expr, "", "exec") + + def test_multi_line_lambda_as_argument(self): + # See gh-101928 + code = textwrap.dedent(""" + def foo(param, lambda_exp): + pass + + foo(param=0, + lambda_exp=lambda: + 1) + """) + compile(code, "", "exec") + + def test_apply_static_swaps(self): + def f(x, y): + a, a = x, y + return a + self.assertEqual(f("x", "y"), "y") + + def test_apply_static_swaps_2(self): + def f(x, y, z): + a, b, a = x, y, z + return a + self.assertEqual(f("x", "y", "z"), "z") + + def test_apply_static_swaps_3(self): + def f(x, y, z): + a, a, b = x, y, z + return a + self.assertEqual(f("x", "y", "z"), "y") + + def test_variable_dependent(self): + # gh-104635: Since the value of b is dependent on the value of a + # the first STORE_FAST for a should not be skipped. (e.g POP_TOP). + # This test case is added to prevent potential regression from aggressive optimization. + def f(): + a = 42; b = a + 54; a = 54 + return a, b + self.assertEqual(f(), (54, 96)) + + def test_duplicated_small_exit_block(self): + # See gh-109627 + def f(): + while element and something: + try: + return something + except: + pass + + def test_cold_block_moved_to_end(self): + # See gh-109719 + def f(): + while name: + try: + break + except: + pass + else: + 1 if 1 else 1 + + def test_remove_empty_basic_block_with_jump_target_label(self): + # See gh-109823 + def f(x): + while x: + 0 if 1 else 0 + + def test_remove_redundant_nop_edge_case(self): + # See gh-109889 + def f(): + a if (1 if b else c) else d + + def test_global_declaration_in_except_used_in_else(self): + # See gh-111123 + code = textwrap.dedent("""\ + def f(): + try: + pass + %s Exception: + global a + else: + print(a) + """) + + g, l = {'a': 5}, {} + for kw in ("except", "except*"): + exec(code % kw, g, l); + + def test_regression_gh_120225(self): + async def name_4(): + match b'': + case True: + pass + case name_5 if f'e': + {name_3: name_4 async for name_2 in name_5} + case []: + pass + [[]] + + def test_globals_dict_subclass(self): + # gh-132386 + class WeirdDict(dict): + pass + + ns = {} + exec('def foo(): return a', WeirdDict(), ns) + + self.assertRaises(NameError, ns['foo']) + + @unittest.expectedFailure # TODO: RUSTPYTHON; + [3, 5, 3, 5] + def test_compile_warnings(self): + # Each invocation of compile() emits compiler warnings, even if they + # have the same message and line number. + source = textwrap.dedent(r""" + # tokenizer + 1or 0 # line 3 + # code generator + 1 is 1 # line 5 + """) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("default") + for i in range(2): + # Even if compile() is at the same line. + compile(source, '', 'exec') + + self.assertEqual([wm.lineno for wm in caught], [3, 5] * 2) + + @unittest.expectedFailure # TODO: RUSTPYTHON; + [5, 9] + def test_compile_warning_in_finally(self): + # Ensure that warnings inside finally blocks are + # only emitted once despite the block being + # compiled twice (for normal execution and for + # exception handling). + source = textwrap.dedent(""" + try: + pass + finally: + 1 is 1 # line 5 + try: + pass + finally: # nested + 1 is 1 # line 9 + """) + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + compile(source, '', 'exec') + + self.assertEqual(sorted(wm.lineno for wm in caught), [5, 9]) + for wm in caught: + self.assertEqual(wm.category, SyntaxWarning) + self.assertIn("\"is\" with 'int' literal", str(wm.message)) + + # Other code path is used for "try" with "except*". + source = textwrap.dedent(""" + try: + pass + except *Exception: + pass + finally: + 1 is 1 # line 7 + try: + pass + except *Exception: + pass + finally: # nested + 1 is 1 # line 13 + """) + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + compile(source, '', 'exec') + + self.assertEqual(sorted(wm.lineno for wm in caught), [7, 13]) + for wm in caught: + self.assertEqual(wm.category, SyntaxWarning) + self.assertIn("\"is\" with 'int' literal", str(wm.message)) + + @unittest.expectedFailure # TODO: RUSTPYTHON + @support.subTests('src', [ + textwrap.dedent(""" + def f(): + try: + pass + finally: + return 42 + """), + textwrap.dedent(""" + for x in y: + try: + pass + finally: + break + """), + textwrap.dedent(""" + for x in y: + try: + pass + finally: + continue + """), + ]) + def test_pep_765_warnings(self, src): + with self.assertWarnsRegex(SyntaxWarning, 'finally'): + compile(src, '', 'exec') + with warnings.catch_warnings(): + warnings.simplefilter("error") + tree = ast.parse(src) + with self.assertWarnsRegex(SyntaxWarning, 'finally'): + compile(tree, '', 'exec') + + @support.subTests('src', [ + textwrap.dedent(""" + try: + pass + finally: + def f(): + return 42 + """), + textwrap.dedent(""" + try: + pass + finally: + for x in y: + break + """), + textwrap.dedent(""" + try: + pass + finally: + for x in y: + continue + """), + ]) + def test_pep_765_no_warnings(self, src): + with warnings.catch_warnings(): + warnings.simplefilter("error") + compile(src, '', 'exec') + + +class TestBooleanExpression(unittest.TestCase): + class Value: + def __init__(self): + self.called = 0 + + def __bool__(self): + self.called += 1 + return self.value + + class Yes(Value): + value = True + + class No(Value): + value = False + + def test_short_circuit_and(self): + v = [self.Yes(), self.No(), self.Yes()] + res = v[0] and v[1] and v[0] + self.assertIs(res, v[1]) + self.assertEqual([e.called for e in v], [1, 1, 0]) + + def test_short_circuit_or(self): + v = [self.No(), self.Yes(), self.No()] + res = v[0] or v[1] or v[0] + self.assertIs(res, v[1]) + self.assertEqual([e.called for e in v], [1, 1, 0]) + + def test_compound(self): + # See gh-124285 + v = [self.No(), self.Yes(), self.Yes(), self.Yes()] + res = v[0] and v[1] or v[2] or v[3] + self.assertIs(res, v[2]) + self.assertEqual([e.called for e in v], [1, 0, 1, 0]) + + v = [self.No(), self.No(), self.Yes(), self.Yes(), self.No()] + res = v[0] or v[1] and v[2] or v[3] or v[4] + self.assertIs(res, v[3]) + self.assertEqual([e.called for e in v], [1, 1, 0, 1, 0]) + + def test_exception(self): + # See gh-137288 + class Foo: + def __bool__(self): + raise NotImplementedError() + + a = Foo() + b = Foo() + + with self.assertRaises(NotImplementedError): + bool(a) + + with self.assertRaises(NotImplementedError): + c = a or b @requires_debug_ranges() class TestSourcePositions(unittest.TestCase): @@ -1118,7 +1884,7 @@ def check_positions_against_ast(self, snippet): class SourceOffsetVisitor(ast.NodeVisitor): def generic_visit(self, node): super().generic_visit(node) - if not isinstance(node, ast.expr) and not isinstance(node, ast.stmt): + if not isinstance(node, (ast.expr, ast.stmt, ast.pattern)): return lines.add(node.lineno) end_lines.add(node.end_lineno) @@ -1147,8 +1913,8 @@ def generic_visit(self, node): def assertOpcodeSourcePositionIs(self, code, opcode, line, end_line, column, end_column, occurrence=1): - for instr, position in zip( - dis.Bytecode(code, show_caches=True), code.co_positions(), strict=True + for instr, position in instructions_with_positions( + dis.Bytecode(code), code.co_positions() ): if instr.opname == opcode: occurrence -= 1 @@ -1186,15 +1952,303 @@ def test_compiles_to_extended_op_arg(self): column=2, end_column=9, occurrence=2) def test_multiline_expression(self): - snippet = """\ -f( - 1, 2, 3, 4 -) -""" + snippet = textwrap.dedent("""\ + f( + 1, 2, 3, 4 + ) + """) compiled_code, _ = self.check_positions_against_ast(snippet) self.assertOpcodeSourcePositionIs(compiled_code, 'CALL', line=1, end_line=3, column=0, end_column=1) + @requires_specialization + def test_multiline_boolean_expression(self): + snippet = textwrap.dedent("""\ + if (a or + (b and not c) or + not ( + d > 0)): + x = 42 + """) + compiled_code, _ = self.check_positions_against_ast(snippet) + # jump if a is true: + self.assertOpcodeSourcePositionIs(compiled_code, 'POP_JUMP_IF_TRUE', + line=1, end_line=1, column=4, end_column=5, occurrence=1) + # jump if b is false: + self.assertOpcodeSourcePositionIs(compiled_code, 'POP_JUMP_IF_FALSE', + line=2, end_line=2, column=5, end_column=6, occurrence=1) + # jump if c is false: + self.assertOpcodeSourcePositionIs(compiled_code, 'POP_JUMP_IF_FALSE', + line=2, end_line=2, column=15, end_column=16, occurrence=2) + # compare d and 0 + self.assertOpcodeSourcePositionIs(compiled_code, 'COMPARE_OP', + line=4, end_line=4, column=8, end_column=13, occurrence=1) + # jump if comparison it True + self.assertOpcodeSourcePositionIs(compiled_code, 'POP_JUMP_IF_TRUE', + line=4, end_line=4, column=8, end_column=13, occurrence=2) + + @unittest.skipIf(sys.flags.optimize, "Assertions are disabled in optimized mode") + def test_multiline_assert(self): + snippet = textwrap.dedent("""\ + assert (a > 0 and + bb > 0 and + ccc == 1000000), "error msg" + """) + compiled_code, _ = self.check_positions_against_ast(snippet) + self.assertOpcodeSourcePositionIs(compiled_code, 'LOAD_COMMON_CONSTANT', + line=1, end_line=3, column=0, end_column=36, occurrence=1) + # The "error msg": + self.assertOpcodeSourcePositionIs(compiled_code, 'LOAD_CONST', + line=3, end_line=3, column=25, end_column=36, occurrence=2) + self.assertOpcodeSourcePositionIs(compiled_code, 'CALL', + line=1, end_line=3, column=0, end_column=36, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'RAISE_VARARGS', + line=1, end_line=3, column=8, end_column=22, occurrence=1) + + def test_multiline_generator_expression(self): + snippet = textwrap.dedent("""\ + ((x, + 2*x) + for x + in [1,2,3] if (x > 0 + and x < 100 + and x != 50)) + """) + compiled_code, _ = self.check_positions_against_ast(snippet) + compiled_code = compiled_code.co_consts[0] + self.assertIsInstance(compiled_code, types.CodeType) + self.assertOpcodeSourcePositionIs(compiled_code, 'YIELD_VALUE', + line=1, end_line=2, column=1, end_column=8, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD', + line=1, end_line=2, column=1, end_column=8, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE', + line=4, end_line=4, column=7, end_column=14, occurrence=1) + + def test_multiline_async_generator_expression(self): + snippet = textwrap.dedent("""\ + ((x, + 2*x) + async for x + in [1,2,3] if (x > 0 + and x < 100 + and x != 50)) + """) + compiled_code, _ = self.check_positions_against_ast(snippet) + compiled_code = compiled_code.co_consts[0] + self.assertIsInstance(compiled_code, types.CodeType) + self.assertOpcodeSourcePositionIs(compiled_code, 'YIELD_VALUE', + line=1, end_line=2, column=1, end_column=8, occurrence=2) + self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE', + line=1, end_line=6, column=0, end_column=32, occurrence=1) + + def test_multiline_list_comprehension(self): + snippet = textwrap.dedent("""\ + [(x, + 2*x) + for x + in [1,2,3] if (x > 0 + and x < 100 + and x != 50)] + """) + compiled_code, _ = self.check_positions_against_ast(snippet) + self.assertIsInstance(compiled_code, types.CodeType) + self.assertOpcodeSourcePositionIs(compiled_code, 'LIST_APPEND', + line=1, end_line=2, column=1, end_column=8, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD', + line=1, end_line=2, column=1, end_column=8, occurrence=1) + + def test_multiline_async_list_comprehension(self): + snippet = textwrap.dedent("""\ + async def f(): + [(x, + 2*x) + async for x + in [1,2,3] if (x > 0 + and x < 100 + and x != 50)] + """) + compiled_code, _ = self.check_positions_against_ast(snippet) + g = {} + eval(compiled_code, g) + compiled_code = g['f'].__code__ + self.assertIsInstance(compiled_code, types.CodeType) + self.assertOpcodeSourcePositionIs(compiled_code, 'LIST_APPEND', + line=2, end_line=3, column=5, end_column=12, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD', + line=2, end_line=3, column=5, end_column=12, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE', + line=2, end_line=7, column=4, end_column=36, occurrence=1) + + def test_multiline_set_comprehension(self): + snippet = textwrap.dedent("""\ + {(x, + 2*x) + for x + in [1,2,3] if (x > 0 + and x < 100 + and x != 50)} + """) + compiled_code, _ = self.check_positions_against_ast(snippet) + self.assertIsInstance(compiled_code, types.CodeType) + self.assertOpcodeSourcePositionIs(compiled_code, 'SET_ADD', + line=1, end_line=2, column=1, end_column=8, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD', + line=1, end_line=2, column=1, end_column=8, occurrence=1) + + def test_multiline_async_set_comprehension(self): + snippet = textwrap.dedent("""\ + async def f(): + {(x, + 2*x) + async for x + in [1,2,3] if (x > 0 + and x < 100 + and x != 50)} + """) + compiled_code, _ = self.check_positions_against_ast(snippet) + g = {} + eval(compiled_code, g) + compiled_code = g['f'].__code__ + self.assertIsInstance(compiled_code, types.CodeType) + self.assertOpcodeSourcePositionIs(compiled_code, 'SET_ADD', + line=2, end_line=3, column=5, end_column=12, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD', + line=2, end_line=3, column=5, end_column=12, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE', + line=2, end_line=7, column=4, end_column=36, occurrence=1) + + def test_multiline_dict_comprehension(self): + snippet = textwrap.dedent("""\ + {x: + 2*x + for x + in [1,2,3] if (x > 0 + and x < 100 + and x != 50)} + """) + compiled_code, _ = self.check_positions_against_ast(snippet) + self.assertIsInstance(compiled_code, types.CodeType) + self.assertOpcodeSourcePositionIs(compiled_code, 'MAP_ADD', + line=1, end_line=2, column=1, end_column=7, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD', + line=1, end_line=2, column=1, end_column=7, occurrence=1) + + def test_multiline_async_dict_comprehension(self): + snippet = textwrap.dedent("""\ + async def f(): + {x: + 2*x + async for x + in [1,2,3] if (x > 0 + and x < 100 + and x != 50)} + """) + compiled_code, _ = self.check_positions_against_ast(snippet) + g = {} + eval(compiled_code, g) + compiled_code = g['f'].__code__ + self.assertIsInstance(compiled_code, types.CodeType) + self.assertOpcodeSourcePositionIs(compiled_code, 'MAP_ADD', + line=2, end_line=3, column=5, end_column=11, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD', + line=2, end_line=3, column=5, end_column=11, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE', + line=2, end_line=7, column=4, end_column=36, occurrence=1) + + def test_matchcase_sequence(self): + snippet = textwrap.dedent("""\ + match x: + case a, b: + pass + """) + compiled_code, _ = self.check_positions_against_ast(snippet) + self.assertOpcodeSourcePositionIs(compiled_code, 'MATCH_SEQUENCE', + line=2, end_line=2, column=9, end_column=13, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'UNPACK_SEQUENCE', + line=2, end_line=2, column=9, end_column=13, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'STORE_NAME', + line=2, end_line=2, column=9, end_column=13, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'STORE_NAME', + line=2, end_line=2, column=9, end_column=13, occurrence=2) + + def test_matchcase_sequence_wildcard(self): + snippet = textwrap.dedent("""\ + match x: + case a, *b, c: + pass + """) + compiled_code, _ = self.check_positions_against_ast(snippet) + self.assertOpcodeSourcePositionIs(compiled_code, 'MATCH_SEQUENCE', + line=2, end_line=2, column=9, end_column=17, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'UNPACK_EX', + line=2, end_line=2, column=9, end_column=17, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'STORE_NAME', + line=2, end_line=2, column=9, end_column=17, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'STORE_NAME', + line=2, end_line=2, column=9, end_column=17, occurrence=2) + self.assertOpcodeSourcePositionIs(compiled_code, 'STORE_NAME', + line=2, end_line=2, column=9, end_column=17, occurrence=3) + + def test_matchcase_mapping(self): + snippet = textwrap.dedent("""\ + match x: + case {"a" : a, "b": b}: + pass + """) + compiled_code, _ = self.check_positions_against_ast(snippet) + self.assertOpcodeSourcePositionIs(compiled_code, 'MATCH_MAPPING', + line=2, end_line=2, column=9, end_column=26, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'MATCH_KEYS', + line=2, end_line=2, column=9, end_column=26, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'STORE_NAME', + line=2, end_line=2, column=9, end_column=26, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'STORE_NAME', + line=2, end_line=2, column=9, end_column=26, occurrence=2) + + def test_matchcase_mapping_wildcard(self): + snippet = textwrap.dedent("""\ + match x: + case {"a" : a, "b": b, **c}: + pass + """) + compiled_code, _ = self.check_positions_against_ast(snippet) + self.assertOpcodeSourcePositionIs(compiled_code, 'MATCH_MAPPING', + line=2, end_line=2, column=9, end_column=31, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'MATCH_KEYS', + line=2, end_line=2, column=9, end_column=31, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'STORE_NAME', + line=2, end_line=2, column=9, end_column=31, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'STORE_NAME', + line=2, end_line=2, column=9, end_column=31, occurrence=2) + + def test_matchcase_class(self): + snippet = textwrap.dedent("""\ + match x: + case C(a, b): + pass + """) + compiled_code, _ = self.check_positions_against_ast(snippet) + self.assertOpcodeSourcePositionIs(compiled_code, 'MATCH_CLASS', + line=2, end_line=2, column=9, end_column=16, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'UNPACK_SEQUENCE', + line=2, end_line=2, column=9, end_column=16, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'STORE_NAME', + line=2, end_line=2, column=9, end_column=16, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'STORE_NAME', + line=2, end_line=2, column=9, end_column=16, occurrence=2) + + def test_matchcase_or(self): + snippet = textwrap.dedent("""\ + match x: + case C(1) | C(2): + pass + """) + compiled_code, _ = self.check_positions_against_ast(snippet) + self.assertOpcodeSourcePositionIs(compiled_code, 'MATCH_CLASS', + line=2, end_line=2, column=9, end_column=13, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'MATCH_CLASS', + line=2, end_line=2, column=16, end_column=20, occurrence=2) + def test_very_long_line_end_offset(self): # Make sure we get the correct column offset for offsets # too large to store in a byte. @@ -1209,16 +2263,16 @@ def test_complex_single_line_expression(self): snippet = "a - b @ (c * x['key'] + 23)" compiled_code, _ = self.check_positions_against_ast(snippet) - self.assertOpcodeSourcePositionIs(compiled_code, 'BINARY_SUBSCR', + self.assertOpcodeSourcePositionIs(compiled_code, 'BINARY_OP', line=1, end_line=1, column=13, end_column=21) self.assertOpcodeSourcePositionIs(compiled_code, 'BINARY_OP', - line=1, end_line=1, column=9, end_column=21, occurrence=1) + line=1, end_line=1, column=9, end_column=21, occurrence=2) self.assertOpcodeSourcePositionIs(compiled_code, 'BINARY_OP', - line=1, end_line=1, column=9, end_column=26, occurrence=2) + line=1, end_line=1, column=9, end_column=26, occurrence=3) self.assertOpcodeSourcePositionIs(compiled_code, 'BINARY_OP', - line=1, end_line=1, column=4, end_column=27, occurrence=3) + line=1, end_line=1, column=4, end_column=27, occurrence=4) self.assertOpcodeSourcePositionIs(compiled_code, 'BINARY_OP', - line=1, end_line=1, column=0, end_column=27, occurrence=4) + line=1, end_line=1, column=0, end_column=27, occurrence=5) def test_multiline_assert_rewritten_as_method_call(self): # GH-94694: Don't crash if pytest rewrites a multiline assert as a @@ -1268,7 +2322,9 @@ def f(): with self.subTest(body): namespace = {} source = textwrap.dedent(source_template.format(body)) - exec(source, namespace) + with warnings.catch_warnings(): + warnings.simplefilter('ignore', SyntaxWarning) + exec(source, namespace) code = namespace["f"].__code__ self.assertOpcodeSourcePositionIs( code, @@ -1314,7 +2370,7 @@ def test_method_call(self): source = "(\n lhs \n . \n rhs \n )()" code = compile(source, "", "exec") self.assertOpcodeSourcePositionIs( - code, "LOAD_METHOD", line=4, end_line=4, column=5, end_column=8 + code, "LOAD_ATTR", line=4, end_line=4, column=5, end_column=8 ) self.assertOpcodeSourcePositionIs( code, "CALL", line=4, end_line=5, column=5, end_column=10 @@ -1345,9 +2401,6 @@ def test_column_offset_deduplication(self): for source in [ "lambda: a", "(a for b in c)", - "[a for b in c]", - "{a for b in c}", - "{a: b for c in d}", ]: with self.subTest(source): code = compile(f"{source}, {source}", "", "eval") @@ -1360,6 +2413,138 @@ def test_column_offset_deduplication(self): list(code.co_consts[1].co_positions()), ) + def test_load_super_attr(self): + source = "class C:\n def __init__(self):\n super().__init__()" + for const in compile(source, "", "exec").co_consts[0].co_consts: + if isinstance(const, types.CodeType): + code = const + break + self.assertOpcodeSourcePositionIs( + code, "LOAD_GLOBAL", line=3, end_line=3, column=4, end_column=9 + ) + + def test_lambda_return_position(self): + snippets = [ + "f = lambda: x", + "f = lambda: 42", + "f = lambda: 1 + 2", + "f = lambda: a + b", + ] + for snippet in snippets: + with self.subTest(snippet=snippet): + lamb = run_code(snippet)["f"] + positions = lamb.__code__.co_positions() + # assert that all positions are within the lambda + for i, pos in enumerate(positions): + with self.subTest(i=i, pos=pos): + start_line, end_line, start_col, end_col = pos + if i == 0 and start_col == end_col == 0: + # ignore the RESUME in the beginning + continue + self.assertEqual(start_line, 1) + self.assertEqual(end_line, 1) + code_start = snippet.find(":") + 2 + code_end = len(snippet) + self.assertGreaterEqual(start_col, code_start) + self.assertLessEqual(end_col, code_end) + self.assertGreaterEqual(end_col, start_col) + self.assertLessEqual(end_col, code_end) + + def test_return_in_with_positions(self): + # See gh-98442 + def f(): + with xyz: + 1 + 2 + 3 + 4 + return R + + # All instructions should have locations on a single line + for instr in dis.get_instructions(f): + start_line, end_line, _, _ = instr.positions + self.assertEqual(start_line, end_line) + + # Expect four `LOAD_CONST None` instructions: + # three for the no-exception __exit__ call, and one for the return. + # They should all have the locations of the context manager ('xyz'). + + load_none = [instr for instr in dis.get_instructions(f) if + instr.opname == 'LOAD_CONST' and instr.argval is None] + return_value = [instr for instr in dis.get_instructions(f) if + instr.opname == 'RETURN_VALUE'] + + self.assertEqual(len(load_none), 4) + self.assertEqual(len(return_value), 2) + for instr in load_none + return_value: + start_line, end_line, start_col, end_col = instr.positions + self.assertEqual(start_line, f.__code__.co_firstlineno + 1) + self.assertEqual(end_line, f.__code__.co_firstlineno + 1) + self.assertEqual(start_col, 17) + self.assertEqual(end_col, 20) + + +class TestStaticAttributes(unittest.TestCase): + + def test_basic(self): + class C: + def f(self): + self.a = self.b = 42 + # read fields are not included + self.f() + self.arr[3] + + self.assertIsInstance(C.__static_attributes__, tuple) + self.assertEqual(sorted(C.__static_attributes__), ['a', 'b']) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: type object 'C' has no attribute '__static_attributes__' + def test_nested_function(self): + class C: + def f(self): + self.x = 1 + self.y = 2 + self.x = 3 # check deduplication + + def g(self, obj): + self.y = 4 + self.z = 5 + + def h(self, a): + self.u = 6 + self.v = 7 + + obj.self = 8 + + self.assertEqual(sorted(C.__static_attributes__), ['u', 'v', 'x', 'y', 'z']) + + def test_nested_class(self): + class C: + def f(self): + self.x = 42 + self.y = 42 + + class D: + def g(self): + self.y = 42 + self.z = 42 + + self.assertEqual(sorted(C.__static_attributes__), ['x', 'y']) + self.assertEqual(sorted(C.D.__static_attributes__), ['y', 'z']) + + def test_subclass(self): + class C: + def f(self): + self.x = 42 + self.y = 42 + + class D(C): + def g(self): + self.y = 42 + self.z = 42 + + self.assertEqual(sorted(C.__static_attributes__), ['x', 'y']) + self.assertEqual(sorted(D.__static_attributes__), ['y', 'z']) + class TestExpressionStackSize(unittest.TestCase): # These tests check that the computed stack size for a code object @@ -1393,28 +2578,22 @@ def test_if_else(self): def test_binop(self): self.check_stack_size("x + " * self.N + "x") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 101 not less than or equal to 6 def test_list(self): self.check_stack_size("[" + "x, " * self.N + "x]") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 101 not less than or equal to 6 def test_tuple(self): self.check_stack_size("(" + "x, " * self.N + "x)") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 101 not less than or equal to 6 def test_set(self): self.check_stack_size("{" + "x, " * self.N + "x}") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_dict(self): self.check_stack_size("{" + "x:x, " * self.N + "x:x}") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 102 not less than or equal to 6 def test_func_args(self): self.check_stack_size("f(" + "x, " * self.N + ")") @@ -1422,8 +2601,7 @@ def test_func_kwargs(self): kwargs = (f'a{i}=x' for i in range(self.N)) self.check_stack_size("f(" + ", ".join(kwargs) + ")") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 102 not less than or equal to 6 def test_meth_args(self): self.check_stack_size("o.m(" + "x, " * self.N + ")") @@ -1460,7 +2638,9 @@ def compile_snippet(i): script = """def func():\n""" + i * snippet if async_: script = "async " + script - code = compile(script, "