diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 22b635337..ed6a25364 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -3,9 +3,9 @@ Thank you for your interest in contributing to virtualenv! There are many ways to contribute, and we appreciate all of them. As a reminder, all contributors are expected to follow the [PSF Code of Conduct][coc]. -[coc]: https://github.com/pypa/.github/blob/main/CODE_OF_CONDUCT.md - ## Development Documentation Our [development documentation](https://virtualenv.pypa.io/en/latest/development.html#development) contains details on how to get started with contributing to `virtualenv`, and details of our development processes. + +[coc]: https://github.com/pypa/.github/blob/main/CODE_OF_CONDUCT.md diff --git a/.github/FUNDING.yaml b/.github/FUNDING.yaml new file mode 100644 index 000000000..bb5081b53 --- /dev/null +++ b/.github/FUNDING.yaml @@ -0,0 +1,3 @@ +github: gaborbernat +tidelift: "pypi/virtualenv" +thanks_dev: u/gh/gaborbernat diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml deleted file mode 100644 index 91b483e4e..000000000 --- a/.github/FUNDING.yml +++ /dev/null @@ -1 +0,0 @@ -tidelift: "pypi/virtualenv" diff --git a/.github/INCIDENT_RESPONSE.md b/.github/INCIDENT_RESPONSE.md new file mode 100644 index 000000000..baef29d1d --- /dev/null +++ b/.github/INCIDENT_RESPONSE.md @@ -0,0 +1,115 @@ +# Incident Response Plan + +How a security problem in virtualenv gets handled once someone reports it. For how to report one, and for what counts as +a vulnerability, read [SECURITY.md](SECURITY.md) instead. + +This is a living document. It changes after each incident that shows it is wrong. + +## What counts as an incident + +A vulnerability in virtualenv itself, or in one of the pip and setuptools wheels bundled under +`src/virtualenv/seed/wheels/embed/`, is the usual case. Three others count and need a different response: + +- A tampered release artifact, on PyPI, in a GitHub Release, or at `bootstrap.pypa.io/virtualenv.pyz`. +- Commits, tags or releases that no maintainer made, or a change to the trusted publisher configuration. +- A compromised maintainer account. + +The last three are less likely than a code bug and worse when they happen, which is why they get their own containment +step below. + +## Who responds + +[Bernát Gábor](https://github.com/gaborbernat) leads the response, because he does most of the day-to-day work and +answers fastest. [Rahul Devikar](https://github.com/rahuldevikar) covers when Bernát is unavailable, and +[Paul Moore](https://github.com/pfmoore) steps in if both are away. Nothing here depends on who is available: anyone +holding release rights can run this plan, and the lead for a given incident is whoever picks it up. + +We are volunteers, with no rotation and no out-of-hours cover. The targets in [SECURITY.md](SECURITY.md) are what this +team can meet on a good week, not a service level. [Tidelift](https://tidelift.com/security) is the lane that carries a +committed acknowledgment, and it handles intake, the first reply and coordination on release timing. Triage, the fix, +the release and the announcement stay here. + +A reporter, or a contributor already trusted with review, may be invited into the private advisory to help build or +check a fix. + +## Severity + +Four levels, set by judgment rather than a calculator. virtualenv is a library and a command line tool, and a score +computed without knowing how a caller invokes it says little. curl, the ASF and OpenSSL all reach the same conclusion +for the same reason, and GitHub's advisory form accepts a plain severity with no vector attached. + +| Level | Meaning | Example from this project | +| -------- | ----------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------- | +| Critical | A tampered artifact reached users, or an account or publishing credential is in someone else's hands. | none so far | +| High | Attacker-chosen data becomes code on a normal path, with no unusual setup. | `GHSA-x78j-v8h9-3j2q`, command injection through `--prompt` into `activate.bat` | +| Medium | Real damage that needs a race, a specific configuration, or local timing to land. | `GHSA-597g-3phw-6986`, a time-of-check to time-of-use gap in directory creation | +| Low | A correctness or integrity failure with a narrow or hard-to-reach effect. | `GHSA-94p9-xgh2-xp45`, seed wheels used without an integrity check | + +Keep undisclosed vulnerabilities in a private advisory and its temporary fork, regardless of severity. High and Critical +need an out-of-band release. Low and Medium may wait for the next release within the agreed disclosure window. A public +pull request exposes the patch even if its title does not mention security. Follow the early-disclosure procedure in +[SECURITY.md](SECURITY.md) if the details are public. + +Check the [threat model](THREAT_MODEL.md) to judge likelihood and reach. It lists the threats we expect at each trust +boundary with their controls, and the risks we chose to accept. + +## Where reports come from + +The [advisory form](https://github.com/pypa/virtualenv/security/advisories/new), the Tidelift contact, Dependabot and +CodeQL, and a public issue that turns out to be worse than it looked and gets moved into a private advisory. +Self-discovery during ordinary work is also a real source: every advisory published against virtualenv so far started +that way. + +## Response + +**Triage.** Confirm the problem, open or adopt a private advisory, set a severity, and decide whether +[SECURITY.md](SECURITY.md) puts it in scope. Reply to the reporter either way. An out-of-scope report gets a reason, not +silence. + +**Contain.** For the artifact, account and credential cases above, stop release jobs and revoke compromised account +sessions, API tokens and deploy keys. Remove a compromised trusted publisher from PyPI until maintainers can restore +trusted access. Keep the GitHub `release` environment and its protection rules; deleting it does not revoke a PyPI +trusted publisher. + +Preserve workflow logs, audit events, artifact hashes and affected commit IDs before cleanup. Yank affected PyPI +releases with a reason, and contact [PyPI security](https://pypi.org/security/) for compromised artifacts. +[Yanking does not prevent installation through an exact version pin](https://docs.pypi.org/project-management/yanking/), +so warn users about affected versions and recovery steps. Check tags against known good commits, restore a verified +`bootstrap.pypa.io/virtualenv.pyz`, and examine releases from the period of compromised access. + +**Measure the blast radius.** For anything beyond a plain code bug, write down whether there is evidence of real +exploitation and how confident that answer is. Guessing in public later is worse than recording uncertainty now. + +**Fix.** Develop the patch and a regression test in the temporary private fork attached to the advisory. Keep the patch, +reproducer and logs within the advisory until disclosure. + +GitHub +[does not run CI or enforce branch protection in temporary private forks](https://docs.github.com/en/code-security/tutorials/fix-reported-vulnerabilities/collaborate-in-a-fork). +Run the relevant tox environments on trusted machines without publishing credentials. Record the commit ID, commands, +platforms and results in the private advisory. Ask another maintainer to review the patch and results before merging; +record any emergency exception and its reason in the advisory. + +**Release.** Follow the usual release process in `docs/development.rst`. Remember there are three surfaces: PyPI, the +GitHub Release zipapp, and `bootstrap.pypa.io/virtualenv.pyz`. + +**Disclose.** Publish the advisory with the release. Request a CVE through GitHub. Credit the reporter as they asked to +be credited. Close any duplicate draft advisories covering the same issue, so the published record has one entry per +problem. + +**Learn.** See below. + +## After the incident + +Within a few days of the release, write a short account in the advisory itself: a timeline, what was affected, the root +cause with a link to the commit or pull request that introduced it, and what the fix changed. Say what worked and what +did not. The account blames the process rather than a person: on a team this small, blaming a person produces nothing +useful and discourages the next write-up. + +Hardening work and process gaps that the incident exposed do not belong in the closed advisory, where nobody will read +them again. They become ordinary public issues. Keeping them separate is what stops the incident from expanding while it +is still open, and stops the follow-up from being quietly dropped once the release ships. + +## Keeping this current + +Review after every incident, and at least once a year even without one. An incident that this plan handled badly is the +best reason to change it, and the only time the gaps are obvious. diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md index 2a1b732f7..1164a2383 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.md +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -1,10 +1,6 @@ ---- -name: Bug report -about: Create a report to help us improve -title: "" -labels: bug -assignees: "" ---- +______________________________________________________________________ + +## name: Bug report about: Create a report to help us improve title: "" labels: bug assignees: "" **Issue** @@ -15,8 +11,11 @@ Describe what's the expected behavior and what you're observing. Provide at least: - OS: + - Shell: + - Python version and path: + - `pip list` of the host python where `virtualenv` is installed: ```console diff --git a/.github/ISSUE_TEMPLATE/feature-request.md b/.github/ISSUE_TEMPLATE/feature-request.md index 8714ceb6e..e9a6b503f 100644 --- a/.github/ISSUE_TEMPLATE/feature-request.md +++ b/.github/ISSUE_TEMPLATE/feature-request.md @@ -1,10 +1,6 @@ ---- -name: Feature request -about: Suggest an enhancement for this project -title: "" -labels: enhancement -assignees: "" ---- +______________________________________________________________________ + +## name: Feature request about: Suggest an enhancement for this project title: "" labels: enhancement assignees: "" **What's the problem this feature will solve?** diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index ae6ea93a0..d36036bb2 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,7 +1,22 @@ ### Thanks for contributing, make sure you address all the checklists (for details on how see [development documentation](https://virtualenv.pypa.io/en/latest/development.html#development)) - [ ] ran the linter to address style issues (`tox -e fix`) + - [ ] wrote descriptive pull request text + - [ ] ensured there are test(s) validating the fix + - [ ] added news fragment in `docs/changelog` folder + - [ ] updated/extended the documentation + +- [ ] for changes to activation scripts, `pyvenv.cfg`, wheel downloads, app-data or release workflows: explained in the + PR text which outside input the change handles, such as a path, a prompt or a downloaded file, and how it stops that + input from running as code. See the + [security scope](https://github.com/pypa/virtualenv/blob/main/.github/SECURITY.md#scope). + +- [ ] ran tests on Python 3.9 and 3.15 (`tox run -e 3.9,3.15`) + +- [ ] linked upstream issues beside any temporary interpreter overrides + +- [ ] kept the PR in draft until CI passes diff --git a/.github/SECURITY.md b/.github/SECURITY.md index 90836425e..3886f91eb 100644 --- a/.github/SECURITY.md +++ b/.github/SECURITY.md @@ -1,13 +1,140 @@ # Security Policy -## Supported Versions +`virtualenv` creates Python environments. A small team maintains it: [Bernát Gábor](https://github.com/gaborbernat) +handles most day-to-day work and answers fastest. [Rahul Devikar](https://github.com/rahuldevikar) covers when Bernát is +unavailable, and [Paul Moore](https://github.com/pfmoore) steps in if both are away. +[Tidelift](https://tidelift.com/security) supports the project. This policy says how to report a security problem, what +happens next, and how long it takes. -| Version | Supported | -| --------- | ------------------ | -| 20.15.1 + | :white_check_mark: | -| < 20.15.1 | :x: | +## Reporting a vulnerability -## Reporting a Vulnerability +**Do not open a public issue for a security problem.** -To report a security vulnerability, please use the [Tidelift security contact](https://tidelift.com/security). Tidelift -will coordinate the fix and disclosure. +Use GitHub's private vulnerability reporting: + +1. Go to [the advisory form](https://github.com/pypa/virtualenv/security/advisories/new). +1. Fill in what you found. Everything stays private until an advisory is published. +1. Submit. The report reaches the maintainers directly. + +If you do not have a GitHub account, or you are a Tidelift subscriber, report through the +[Tidelift security contact](https://tidelift.com/security) instead. Tidelift coordinates the fix and the disclosure with +us. If you subscribe to Tidelift, prefer that path: it carries a contractual response time the project cannot offer on +its own. + +If neither path works for you, email [gaborjbernat@gmail.com](mailto:gaborjbernat@gmail.com). + +### What to include + +- What the problem is and what an attacker gains from it. +- The steps or a proof of concept that reproduce it. +- The versions, operating system and Python you tested against. +- Any mitigation you already know of. +- A severity estimate and the CWE that fits best; the form requires at least one CWE. + +Write the report yourself. Reports that consist of unreviewed tool or model output take longer to triage, and we may +close them without a detailed reply. + +## What to expect + +We are volunteers, with no on-call rotation and no 24-hour incident response. The targets are an acknowledgment within +**5 working days** and an assessment within **15 working days**. Those are targets and not guarantees. Tidelift +subscribers get a committed response time through the Tidelift lane. + +If two weeks pass with no acknowledgment at all, escalate through the +[Tidelift security contact](https://tidelift.com/security). + +## Supported versions + +Only the most recent release is supported. Fixes ship in a new release rather than as backports to older lines. + +virtualenv follows a backwards-compatible release policy, so upgrading to the newest version is usually a small change. +Before reporting, confirm the problem still reproduces on the latest release. + +## Scope + +virtualenv takes values from whoever runs it, such as the destination path and `--prompt`, and from the network, such as +the seed wheels it downloads. It turns those values into shell scripts, into `pyvenv.cfg`, and into files on disk. That +conversion is the boundary this policy cares about. + +**In scope** + +- Caller-supplied values changing the meaning of a generated script rather than appearing as data, in any activator: + `activate`, `activate.bat`, `activate.csh`, `activate.fish`, `activate.nu`, `Activate.ps1`, `activate.xsh`. +- The same problem in `pyvenv.cfg` or any other file virtualenv writes. +- Missing or incorrect integrity checking of anything virtualenv downloads, including seed wheels and app-data cache + entries. +- virtualenv writing outside the directory it was pointed at, whether through a symlink, a race or path handling. +- Files or directories created with wider permissions than intended. + +**Out of scope** + +- An attacker who already has arbitrary command execution as the victim, or can replace the victim's executables or + app-data contents. This does not exclude an attacker who controls only a value that a caller passes to virtualenv, + such as project metadata used as a prompt or destination path. +- Tricking a user into executing shell syntax in the command they use to launch virtualenv. Injection from argument data + into a generated activation script or configuration file remains in scope. +- Code that runs from packages virtualenv seeds or that are installed into the environment afterwards. Report malware on + PyPI to [PyPI security](https://pypi.org/security/). +- Vulnerabilities in pip or setuptools themselves. Report those to their own projects. +- Test code and test fixtures. +- Unreleased code on `main`. +- Anything reachable only on a Python version virtualenv no longer supports. + +If you cannot tell which side a finding falls on, report it and we will assess it. + +The [threat model](THREAT_MODEL.md) gives the reasoning behind these lists. It covers what virtualenv trusts, its trust +boundaries and the current controls, and its [assurance case](THREAT_MODEL.md#assurance-case) maps each in-scope item to +the code and tests that enforce it, with the date of our last review. On a conflict we follow this policy and fix the +threat model. + +## How a report is handled + +1. **Triage.** Confirm the report, decide whether it is a vulnerability under the scope above, and set a severity. +1. **Fix in private.** We work in a temporary private fork attached to the draft advisory. +1. **Embargo.** The default window is **90 days** from the report. It shrinks when a problem is being exploited and can + grow when a fix turns out to be hard. We agree any change with the reporter. +1. **Identifier.** We request a CVE through GitHub once a fix exists. +1. **Release.** The fix ships in a new release, and we publish the advisory alongside it. + +## Credit + +We credit reporters in the advisory under whatever name or alias they ask for, and they can stay anonymous. GitHub sends +a credit invitation that the reporter has to accept, so a name appears only with their agreement. + +## If the problem becomes public early + +The embargo ends when the details are public. At that point the advisory goes out with whatever mitigation exists, and a +fix follows as soon as one is ready. Advance warning that a report is about to be published is welcome and far better +than finding out from the timeline. + +## Bug bounty + +There is none. This project does not pay for reported vulnerabilities. + +## Published advisories + +Past advisories are at [the project's advisory list](https://github.com/pypa/virtualenv/security/advisories). To be +notified of new ones, open the repository page and choose **Watch**, then **Custom**, then enable **Security alerts**. + +## EU Cyber Resilience Act + +The [Cyber Resilience Act](https://eur-lex.europa.eu/eli/reg/2024/2847/oj) (Regulation (EU) 2024/2847) sets security +requirements for products with digital elements placed on the EU market. virtualenv is open source software under the +MIT License, developed in public by volunteers under the [Python Packaging Authority](https://www.pypa.io/). We do not +sell virtualenv, support contracts for it or paid features. Maintainers accept sponsorship through GitHub Sponsors and +thanks.dev, and Tidelift supports the project as described at the top of this policy. + +This section says what the project does and where to find its artifacts. It is not a legal assessment of the role +virtualenv, the PyPA or anyone else holds under the regulation. The OpenSSF +[CRA brief guide for open source developers](https://best.openssf.org/CRA-Brief-Guide-for-OSS-Developers.html) explains +those roles. + +If you ship virtualenv in a product and need supply-chain information for your own obligations, it is here: + +- **Release artifacts.** [Release artifacts](https://virtualenv.pypa.io/en/latest/reference/release-artifacts.html) + lists what each release publishes, including the CycloneDX SBOM embedded in every wheel and the attestations. + [Verify a virtualenv release](https://virtualenv.pypa.io/en/latest/how-to/verify-release.html) shows how to check + them. +- **Vulnerabilities.** Report them as described in [Reporting a vulnerability](#reporting-a-vulnerability). Published + advisories are at . +- **Support period.** See [Supported versions](#supported-versions): fixes ship only in the latest release. diff --git a/.github/THREAT_MODEL.md b/.github/THREAT_MODEL.md new file mode 100644 index 000000000..d89d4e152 --- /dev/null +++ b/.github/THREAT_MODEL.md @@ -0,0 +1,565 @@ +# Threat Model and Assurance Case + +This document has two parts. The threat model lists what virtualenv trusts, its trust boundaries, the threats we expect +at each boundary and the current controls. The [assurance case](#assurance-case) at the end ties each control to its fix +and the tests behind it. [SECURITY.md][security] covers [reporting][security-report] through +[private vulnerability reporting][gh-pvr] and defines a vulnerability, and [INCIDENT_RESPONSE.md][ir] covers our +[response][ir-response] after a report. If this document contradicts SECURITY.md, we follow SECURITY.md and correct this +document. + +We based the layout on the GitHub Secure Open Source Fund [threat modeling template for libraries][sosf-template] and +group threats by [STRIDE], one of the methods the [OWASP threat modeling guide][owasp-tm] covers. We took the split +between project promises and caller duties from the [Apache Commons][commons] and [Apache Shiro][shiro] models. + +## Overview + +virtualenv creates Python virtual environments, and [How virtualenv works][expl-how] explains the design. People run it +four ways: the `virtualenv` command, `python -m virtualenv`, the zipapp from a [GitHub Release][gh-releases] or +`https://bootstrap.pypa.io/virtualenv.pyz`, and the [Python API][api]. [tox] and [hatch] call the API or the command +with values they read from project configuration. + +A run has four stages. + +1. [Discovery][expl-discovery] finds and runs the target Python interpreter to learn its layout, and caches the answer + in the [app-data directory][cli-app-data]. +1. [Creation][expl-creators] writes the environment directory with interpreter links or copies, + [`pyvenv.cfg`][src-pyenv-cfg], `CACHEDIR.TAG` and `.gitignore`, as [environment layout][env-layout] describes. +1. [Seeding][expl-seeders] installs pip and setuptools from wheels. virtualenv takes the wheels from the + [copies embedded in its package][src-embed-dir], from newer wheels a [periodic update][expl-periodic] downloaded into + app-data, from directories passed with [`--extra-search-dir`][cli-extra-search-dir], or from PyPI when the user sets + [`--download`][cli-download]. +1. [Activation][expl-activators] renders one [script per shell][src-activation] (bash, batch, csh, fish, Nushell, + PowerShell, xonsh and Python) from templates and substitutes the destination path, the [prompt][cli-prompt] and the + interpreter's Tcl/Tk paths. + +Discoverers, creators, seeders and activators are [plugins][plugin-docs]. virtualenv [loads them][src-plugin-base] from +the `virtualenv.discovery`, `virtualenv.create`, `virtualenv.seed` and `virtualenv.activate` entry point groups of each +installed distribution. + +## Scope + +This model covers the released virtualenv package and zipapp, the files a run writes, the network requests a run makes, +and the pipeline that builds and publishes releases. pip, setuptools and the packages a user adds to an environment fall +outside it; those projects keep their own threat models. We exclude test code, fixtures and unreleased code on `main` +from this model and from the [SECURITY.md scope][security-scope]. + +## Trusted components + +virtualenv runs with the full rights of the user who starts it and does what that user asks, so we draw no trust +boundary at the public API or the command line. An attacker who controls one of the components below is outside this +model. + +- The invoking user and their environment, meaning command line options, `VIRTUALENV_*` + [environment variables][usage-env], the `virtualenv.ini` [configuration file][usage-config-location] in the user + configuration directory or at `VIRTUALENV_CONFIG_FILE`, and [pip's own configuration][pip-config] (`PIP_INDEX_URL`, + `pip.conf`). +- The target interpreter. virtualenv executes it during discovery and creation, and the environment keeps pointing at + it. A malicious interpreter has the user's rights with or without virtualenv. +- The filesystem at the destination. We assume no other user can write to the destination's parent directories. + virtualenv refuses destinations it cannot write to, resolves the path before use, and since + [GHSA-597g-3phw-6986][ghsa-597g] creates directories with atomic calls in place of check-then-create. It does not + defend against another user racing it inside a directory both can write. +- The [app-data directory][src-app-data], a per-user cache directory from [platformdirs] unless the user picks another + with [`--app-data`][cli-app-data] or [`VIRTUALENV_OVERRIDE_APP_DATA`][usage-override-app-data]. It holds downloaded + wheels, extracted wheel images, cached interpreter information and the periodic update log. virtualenv reads these + back without re-verifying them. +- Installed plugins. virtualenv imports each entry point in the four groups above on start-up, selected or not. A + built-in entry point wins over a third-party one with the same name. +- PyPI over verified TLS. For seed wheels that `--download` or the [periodic update][src-periodic-update] fetches, pip + downloads from the index in its configuration, and virtualenv compares the file's SHA-256 with the digest the + [PyPI JSON API][pypi-json] publishes for that filename. On a mismatch virtualenv aborts before it caches or uses the + wheel. If PyPI has no digest for the filename or the metadata request fails, virtualenv retries and refuses the wheel + when the digest stays unavailable ([#3302][pr-3302]). For the bundled version it seeds the bundled wheel in its place, + a pinned version such as `--pip 23.1` fails with an error, and a periodic update keeps the current wheel. virtualenv + skips the check when the environment sets `PIP_INDEX_URL`, `PIP_EXTRA_INDEX_URL` or `PIP_INDEX`. virtualenv cannot see + an index set in `pip.conf`, so it runs the PyPI check on those downloads and refuses a filename PyPI does not know. +- Directories passed with [`--extra-search-dir`][cli-extra-search-dir]. virtualenv uses the wheels it finds there as + given, with no hash check, because the user chose that source. + +virtualenv treats the content of caller-passed values as untrusted. A destination path or [`--prompt`][cli-prompt] may +come from project metadata the user did not write. Those values must reach `pyvenv.cfg` and the generated scripts as +data and must not change the meaning of the file. We put this boundary in scope in [SECURITY.md][security-scope]. + +## Assets + +- Integrity of generated files. The user's shell sources an activation script, and the interpreter and other tools read + `pyvenv.cfg`. Code an attacker injects into either runs as the user. +- Integrity of seeded wheels. pip and setuptools run with the user's rights in each environment virtualenv creates. +- Integrity of released artifacts, meaning the sdist and wheel on [PyPI][pypi-project], the zipapp and SBOMs on + [GitHub Releases][gh-releases], and the zipapp that `bootstrap.pypa.io` serves from + [pypa/get-virtualenv][get-virtualenv]. [Release artifacts][ra-files] lists each file. tox and hatch depend on + virtualenv, so a tampered release reaches tox and hatch users as well as direct users. +- Maintainer and publishing credentials, meaning GitHub accounts with admin rights on `pypa/virtualenv`, PyPI owner + accounts, the PyPI [trusted publisher][pypi-tp] configuration, and the private key of the + [release GitHub App][release-app] in the `release` [deployment environment][gh-environments]. +- The project's reputation, which downstream projects rely on when they depend on virtualenv without pinning it. + +## Entry points and trust boundaries + +The first diagram shows a run of virtualenv, the second how a release reaches users. Dashed arrows cross a trust +boundary. + +```mermaid +flowchart LR + subgraph user["Invoking user (trusted)"] + cli["CLI args and API calls"] + env["VIRTUALENV_* env vars"] + ini["virtualenv.ini"] + end + caller["Caller-supplied values
destination, --prompt"] + subgraph proc["virtualenv process"] + plugins["Plugins via entry points"] + disc["Discovery"] + creator["Creator"] + seeder["Seeder"] + act["Activators"] + end + interp["Target interpreter"] + appdata[("App-data directory")] + pypi["PyPI
wheels and JSON API"] + subgraph dest["Destination directory"] + cfg["pyvenv.cfg"] + scripts["Activation scripts"] + pkgs["Seeded pip, setuptools"] + end + cli --> disc + env --> disc + ini --> disc + caller -.->|"TB1 quoting"| creator + caller -.->|"TB1 quoting"| act + plugins --> disc + disc -->|runs| interp + disc <--> appdata + seeder <--> appdata + pypi -.->|"TB2 TLS + SHA-256 check"| appdata + creator --> cfg + act --> scripts + seeder --> pkgs + classDef trusted fill:#cfe4ff,stroke:#1f6feb,color:#0b1f3a; + classDef external fill:#ffe3a3,stroke:#d29200,color:#3a2c00; + classDef output fill:#d3f4d1,stroke:#2da44e,color:#0f2e14; + class cli,env,ini,interp,appdata,plugins trusted; + class caller,pypi external; + class cfg,scripts,pkgs output; +``` + +```mermaid +flowchart LR + maint["Maintainer"] + main["main branch
ruleset: pull request, checks"] + pre["pre-release.yaml
release commit and tag"] + tag["Tag *.*.*
tag ruleset"] + build["release.yaml build
no publishing credentials"] + publish["release.yaml publish
environment: release"] + verify["release.yaml verify jobs
no publishing credentials"] + pypi[("PyPI
sdist, wheel, attestations")] + ghr[("GitHub Release
zipapp, provenance, SBOMs")] + getv[("pypa/get-virtualenv")] + boot["bootstrap.pypa.io/virtualenv.pyz"] + users["Users and downstream tools"] + maint -.->|"TB4 pull request"| main + maint -->|dispatch| pre + main --> pre + pre -->|release App token| tag + tag --> build + build -->|artifacts| publish + publish -.->|"TB3 trusted publishing"| pypi + publish -.->|"TB3 GITHUB_TOKEN"| ghr + publish -.->|"TB3 release App token"| getv + getv --> boot + pypi --> verify + ghr --> verify + boot --> verify + pypi --> users + ghr --> users + boot --> users + classDef trusted fill:#cfe4ff,stroke:#1f6feb,color:#0b1f3a; + classDef external fill:#ffe3a3,stroke:#d29200,color:#3a2c00; + classDef output fill:#d3f4d1,stroke:#2da44e,color:#0f2e14; + class maint,main,pre,tag,build,publish,verify trusted; + class users external; + class pypi,ghr,getv,boot output; +``` + +| Boundary | Crossing | Controls | +| -------- | -------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| TB1 | Caller-supplied values into activation scripts and `pyvenv.cfg` | [Per-shell quoting][src-activation], [line boundary collapsing][src-text], [property tests][tests-property] and [fuzzing][atheris] | +| TB2 | Wheel bytes from an index through app-data into an environment | pip's TLS, SHA-256 comparison with PyPI's published digest, refusal when PyPI has none ([#3302][pr-3302]) | +| TB3 | Built artifacts from GitHub Actions to PyPI, GitHub Releases, get-virtualenv and bootstrap.pypa.io | [Trusted publishing][pypi-tp], [attestations][gh-attestations], [immutable releases][gh-immutable], [post-publish verification][workflow-release] of each destination | +| TB4 | Code from contributors and dependencies into a release | [Rulesets][gh-rulesets], [SHA-pinned actions][gh-sha-pinning], [embedded wheel hashes][src-embed-init], [wheel age gate][task-wheel-age], [pinned CI tool downloads][ci-tools], [zipapp lock][pylock-zipapp] | + +## Threats and existing mitigations + +We rate risk from likelihood and impact. High means likely and severe, Medium needs a specific setup or a less likely +event, and Low means unlikely or narrow in effect. We use the rating to order our own work and set the severity of a +reported vulnerability through [INCIDENT_RESPONSE.md][ir-severity]. Rows within each [STRIDE] group run from highest +risk to lowest. + +### Spoofing + +| ID | Threat | Risk | Mitigation | +| --- | --------------------------------------------------------------------------------------------------------------- | ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| S1 | An attacker takes over a maintainer's GitHub or PyPI account and publishes a release. | High | PyPI uploads use [trusted publishing][pypi-tp] from [release.yaml][workflow-release], so no long-lived PyPI token exists to steal. PyPI [requires 2FA][pypi-2fa] on each account. A [tag ruleset][gh-rulesets] reserves creating, moving and deleting tags for admins. Artifacts carry [attestations][gh-attestations] that tie them to the workflow, and [immutable releases][gh-immutable] block changes to GitHub release assets after publication. | +| S2 | An attacker registers a lookalike name on PyPI (`virtualenvs`, `virtual-env`) to catch users who mistype. | Medium | PyPI resolves names that normalize to `virtualenv` under [PEP 503][pep-503] to this project. Other lookalikes are outside our control; we report them to [PyPI security][pypi-security]. | +| S3 | A network attacker or a compromised mirror serves a different wheel under a genuine pip or setuptools filename. | Low | pip verifies TLS. virtualenv [compares the downloaded file's SHA-256][src-periodic-update] with PyPI's record and refuses a mismatch ([GHSA-94p9-xgh2-xp45][ghsa-94p9]) or a wheel PyPI lists no digest for ([#3302][pr-3302]). Unless the user sets [`VIRTUALENV_PERIODIC_UPDATE_INSECURE`][usage-insecure], the PyPI metadata request has no unverified TLS fallback. | +| S4 | A third-party package registers an entry point named like a built-in activator or creator to replace it. | Low | Built-in entry points win on a name collision ([plugin loading][src-plugin-base]). A package that can register entry points runs code at import, so the collision gives it no new capability. | + +### Tampering + +| ID | Threat | Risk | Mitigation | +| --- | -------------------------------------------------------------------------------------------------------------------------- | ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| T1 | An attacker puts shell syntax in a destination path, prompt or interpreter path, and an activation script runs it as code. | High | Each [activator][src-activation] quotes caller values with rules for its shell. POSIX shells use the standard library's shell quoting, csh adds escaping for history and prompt expansion, and PowerShell, Nushell, xonsh and Python have their own rules. The [batch activator][src-batch] doubles percent signs, replaces cmd.exe operators and line breaks, and skips generation for paths it cannot represent. [Hypothesis] [property tests][tests-batch-quote] cover batch quoting, and [Atheris] fuzz targets cover [PowerShell][fuzz-powershell] and [Nushell][fuzz-nushell] quoting; [CI][workflow-check] runs both. Two advisories came from this class, [GHSA-x78j-v8h9-3j2q][ghsa-x78j] in the batch activator and [GHSA-p58f-9548-mpm2][ghsa-p58f] in the bash and fish activators. | +| T2 | A caller value adds a line or a key to `pyvenv.cfg`. | Medium | The [`pyvenv.cfg` writer][src-pyenv-cfg] [collapses line boundaries][src-text] in keys and values before writing ([GHSA-9h9j-4vrj-gf7g][ghsa-9h9j]). [Property tests][tests-pyenv-cfg] check that a written file keeps one line per key and gains no new key, and a [fuzz target][fuzz-pyenv-cfg] exercises the same writer. | +| T3 | virtualenv embeds a compromised pip or setuptools release from PyPI. | Medium | The daily [upgrade workflow][workflow-upgrade] [skips wheels uploaded to PyPI in the last seven days][task-wheel-age] and opens a pull request for a maintainer to review. PyPI and the upstream project get that week to catch a bad upload. The workflow records whatever SHA-256 PyPI served, so we would hash and ship a malicious upstream release that stays up seven days like any other. | +| T4 | A malicious pull request or a compromised GitHub Action alters a release. | Medium | The `main` [ruleset][gh-rulesets] requires a pull request and passing status checks across the CI matrix, and it blocks force pushes and deletion. We pin each action to a commit SHA and each [pre-commit hook][precommit-config] to a frozen commit, [Dependabot][dependabot-config] proposes updates after a seven-day [cooldown][dependabot-cooldown], and [zizmor] and [CodeQL][workflow-codeql] scan the workflows. CI checks the RustPython and Nushell downloads against SHA-256 values in [ci-tools.json][ci-tools], which the upgrade workflow refreshes each week ([#3293][pr-3293]). The release workflows install uv, the build backend and every tool they run from [release-requirements.txt][release-lock], a hash-pinned lock that the upgrade workflow refreshes each day with a seven-day cooldown. The build job has read-only repository access and no publishing credentials, its checkout does not persist credentials, and [harden-runner] blocks outbound traffic outside a per-job allowlist in each release, pre-release and upgrade job. It also removes sudo and container access from each of those jobs except PyPI publishing, which runs in a container. No workflow uses [`pull_request_target`][gh-prt], so pull requests from forks run without secrets. The ruleset asks for one approval covering the last push, but admins bypass it, so we do not count a second reviewer as a control. | +| T5 | A compromised release of a runtime dependency (distlib, filelock, platformdirs, python-discovery) reaches users. | Medium | The wheel and sdist declare [version ranges][pyproject], and each user's installer resolves them, so we cannot pin for them. [Dependabot][dependabot-config] checks them each week with a seven-day cooldown. The zipapp bundles the versions a [PEP 751][pep-751] lock pins by SHA-256, and the upgrade workflow refreshes the lock after the same seven-day cooldown ([#3306][pr-3306]). | +| T6 | Someone modifies embedded wheels after the release build, on disk or inside the zipapp. | Low | The [embed module][src-embed-init] records the SHA-256 of each embedded wheel and checks it before first use in each process, reading from inside the zipapp when needed. The [wheel build][hatch-build] generates the embedded [CycloneDX] SBOM from the same table. | +| T7 | Another local user redirects virtualenv's writes through a symlink or a race. | Low | We replaced check-then-create with atomic directory creation ([GHSA-597g-3phw-6986][ghsa-597g]). The [creator][src-creator] resolves the destination and checks write access before use. Directories shared with other users stay outside what we can defend; see accepted risks. | + +### Repudiation + +| ID | Threat | Risk | Mitigation | +| --- | --------------------------------------------------------------------------- | ---- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| R1 | A user cannot tell whether a published artifact came from the project's CI. | Low | [PyPI attestations][pypi-attestations] cover the sdist and wheel, and a [SLSA build provenance][slsa-provenance] attestation ships beside the zipapp. [GitHub attestations][gh-attestations] bind the CycloneDX SBOM and its [SPDX] rendering to both distributions ([#3299][pr-3299]) and the zipapp SBOM to the zipapp ([#3310][pr-3310]). The wheel carries its SBOM as [PEP 770][pep-770] describes. GitHub signs a release attestation over the assets of each immutable release. The release pins timestamps to [`SOURCE_DATE_EPOCH`][sde] and the zipapp's bundled wheels to a [PEP 751][pep-751] lock, and the SBOMs record the build tools, so anyone can [rebuild the sdist, wheel and zipapp][verify-rebuild] and compare the bytes ([#3306][pr-3306], [#3311][pr-3311]). [Verify a virtualenv release][verify] shows each check. We do not require signed commits. | + +### Information disclosure + +| ID | Threat | Risk | Mitigation | +| --- | ------------------------------------------------------------------------- | ---- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| I1 | A publishing credential leaks through a workflow log, artifact or commit. | Low | Trusted publishing leaves no PyPI token to leak. Workflows start with no token permissions or read-only access and grant more per job. We enabled [secret scanning][gh-secret-scanning] and [push protection][gh-push-protection]. [zizmor] flags template injection and credential persistence. | +| I2 | virtualenv exposes local details. | Low | By design, `pyvenv.cfg` records the interpreter path and the command that created the environment. The [periodic update][expl-periodic] contacts PyPI at a [fixed interval][src-periodic-update] unless the user passes [`--no-periodic-update`][cli-no-periodic-update]. | + +### Denial of service + +| ID | Threat | Risk | Mitigation | +| --- | ---------------------------------------------------------------------------------------------------------------- | ---- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| D1 | A PyPI outage or a failing download blocks environment creation. | Low | [`--download`][cli-download] is off by default, so seeding needs no network. The periodic update runs in a detached background process, and its failure leaves the current run untouched. | +| D2 | The release workflow publishes to some destinations and fails on others, and users end up on different versions. | Low | A [recovery job][workflow-release] reports partial publication, and the [development guide][dev-recovery] describes completing it with the original verified artifacts. | +| D3 | bootstrap.pypa.io serves an older zipapp than the latest release. | Low | The release workflow's bootstrap verification job downloads each zipapp path on bootstrap.pypa.io, compares it byte for byte with the release asset, verifies its attestation, and fails the run when the served copy does not catch up ([#3309][pr-3309]). A [pypa/bootstrap][pypa-bootstrap] monitor failure left an older release on bootstrap.pypa.io until [pypa/bootstrap#12][bootstrap-12] fixed it. | + +### Elevation of privilege + +| ID | Threat | Risk | Mitigation | +| --- | ------------------------------------------------------------------------------------------------------------------------- | ---- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| E1 | An attacker with write access to the app-data directory swaps a cached wheel, and the wheel runs in each new environment. | Low | The default location is private to the user. virtualenv does not re-hash cached wheels on use, so an attacker with write access to this directory succeeds; we list that attacker as out of scope in [SECURITY.md][security-scope]. | +| E2 | A privileged user runs virtualenv on a path or app-data directory a less privileged user controls. | Low | We add no defense beyond T7; see accepted risks. | + +## Library and supply chain threats + +These threats target the project and its releases. They weigh more on a package that tox and hatch pull in without +pinning. + +S1 covers maintainer account takeover. Repository admins and the [release App][release-app] bypass both +[rulesets][gh-rulesets], so an admin account can push to `main`, create a tag and trigger a release with no other +account involved, and the attestations would show a genuine workflow run of attacker-chosen code. +[INCIDENT_RESPONSE.md][ir-response] covers containment. + +A patient contributor can earn trust and land a harmful change, as in the [xz backdoor][xz]. CI runs on each pull +request, and a non-admin cannot merge without passing checks. Bernát reviews most pull requests without a second +reviewer, so review rests on one person's judgment; the [current maintainers][dev-maintainers] section lists who covers +when Bernát is away. Our [AI-assisted contribution rules][dev-ai] make the submitter responsible for each line, and we +close submissions with no human in the loop. + +T3, T5 and T6 cover compromised dependencies and embedded wheels, and S2 covers registry typosquatting. Tools that +install virtualenv unpinned take each new release on the day PyPI publishes it. We cannot slow that down for them, so we +keep releases verifiable with attestations, an sdist, wheel and zipapp that [rebuild byte for byte][ra-repro], SBOMs and +[published advisories][advisories]. The [CRA section][security-cra] of SECURITY.md points integrators to the same +material, and the project reports its practices through [OpenSSF Scorecard][scorecard-virtualenv] and the +[OpenSSF Best Practices badge][bestpractices-virtualenv]. + +## Accepted risks + +We know about these risks and chose to keep them. We answer reports that restate one of them with a link to this +section. + +- The maintainers left the `release` [deployment environment][gh-environments] without protection rules. With one active + maintainer, a required reviewer adds a click that the same person grants, and a token with repository scope can + approve deployments through the API. A tag or branch policy does not stop a compromised maintainer token, because + pushing a matching `*.*.*` tag starts the release. We rely on trusted publishing and attestations. +- Admins bypass the `main` and tag [rulesets][gh-rulesets]. The [pre-release workflow][workflow-pre-release] pushes the + release commit to `main` and creates the tag with a token from the [release App][release-app], which the rulesets list + as a bypass actor next to admins. +- A root or service account that runs virtualenv on a destination, app-data directory or temporary directory that a less + privileged user can write gives that user a way in. Use a directory that no other user can write. +- Seeded pip and setuptools run as the user, and so does each package the user adds to the environment. virtualenv does + not inspect or sandbox them. Report malicious packages to [PyPI security][pypi-security]. +- A user who installs a package that registers a virtualenv entry point trusts that package with full rights. +- virtualenv skips the PyPI digest check for custom indexes set through environment variables. It does not re-hash + wheels from `--extra-search-dir` or the app-data cache. [`VIRTUALENV_PERIODIC_UPDATE_INSECURE`][usage-insecure] + restores an unverified TLS fallback for whoever sets it. + +## Open items + +We would accept a fix for each of these gaps, and we track each one in a public pull request or issue once someone picks +it up. + +- virtualenv does not tie an embedded wheel to an attestation from the pip or setuptools project; the hash table records + what PyPI served on the upgrade day. + +## Update triggers + +Update both parts in the pull request that makes one of these changes: + +- adds a source of input, a file virtualenv writes, or a network request; +- changes how virtualenv acquires, verifies or caches wheels, or where app-data lives; +- changes the release workflows, the environments, the rulesets, the trusted publisher configuration or who holds + release rights; +- adds a runtime dependency, an artifact or a distribution channel. + +Review this document with [INCIDENT_RESPONSE.md][ir-after] after each incident, and no more than a year after the last +review. If a published advisory falls in a class this document does not list, add the class. + +## Assurance case + +In this part we argue that virtualenv meets its stated security requirements and list the evidence for each step. We +take the threats, trust boundaries and gaps from the threat model above, and tie each control to its code and to the +tests that keep the fix in place. + +### Review record + +[Bernát Gábor](https://github.com/gaborbernat) reviewed this part on 2026-09-22 against the +[21.10.0 release][release-21.10.0], and updated it for the pull requests merged after that release: [#3293][pr-3293], +[#3299][pr-3299], [#3302][pr-3302], [#3305][pr-3305], [#3309][pr-3309] and [#3310][pr-3310]. For each requirement below +Bernát read the code that enforces it, the pull request and advisory that introduced the fix, and the tests that +exercise it, and confirmed that each linked file exists on `main`. Bernát checked the workflows and the repository's +Actions settings behind the least privilege claims. + +Bernát wrote and reviewed this part, so it carries the blind spots of one reader. Rahul Devikar covers when Bernát is +unavailable, and Paul Moore is the backup if both step away; the [current maintainers][dev-maintainers] section lists +them. If you find a claim here that the code does not support, report it as [SECURITY.md][security-report] describes. + +We repeat the review on the [update triggers](#update-triggers), after each published advisory, and within twelve months +of the previous review. + +### Security requirements + +We treat the in-scope items in [SECURITY.md][security-scope] as the requirements. Caller values stay data in activation +scripts and in `pyvenv.cfg` and other written files, virtualenv checks the integrity of downloads, writes stay inside +the named directory, and permissions stay no wider than intended. The two caller-value requirements sit on TB1 and the +download requirement on TB2; the write and permission requirements apply wherever virtualenv touches the filesystem. We +argue that the release above meets them, apart from the exceptions under [Accepted risks](#accepted-risks) and +[Open items](#open-items). + +### Secure design principles + +We apply least privilege in the workflows and in the tool. Each workflow except [upgrade.yaml][workflow-upgrade] starts +with no token permissions and grants each job the permissions it needs and no more; upgrade.yaml starts with read access +to contents. The repository's [default workflow token][gh-token-permissions] is read-only, and GitHub +[rejects actions not pinned to a full commit SHA][gh-sha-pinning]. Checkouts do not persist credentials, except in the +upgrade publish job, which pushes the upgrade branch with the workflow token. [pre-release.yaml][workflow-pre-release] +builds the changelog in a job with no credentials, from tools that a hash-pinned lock fixes, and a second job mints the +release App token to create the signed release commit and tag through the GitHub API. The commit takes only +`docs/changelog.rst` from the first job. PyPI uploads use trusted publishing (S1). In the tool, the app-data seeder +[marks the extracted wheel image read-only][src-symlink] when it links packages by symlink. + +virtualenv uses fail-safe defaults on the network and in the batch activator. If the TLS handshake with the PyPI +metadata API fails, virtualenv does not retry without verification; the unverified fallback needs the +[opt-in variable][usage-insecure] from S3 ([#3122][pr-3122]). virtualenv refuses a seed wheel with a mismatched or +missing SHA-256 before caching or using it, and seeds the bundled wheel where one exists ([#3302][pr-3302]). The +[batch activator][src-batch] logs a warning and writes no batch scripts for a path it cannot quote without changing it +([#3280][pr-3280]). + +Complete mediation holds at the two TB1 outputs. Each caller value that enters a generated script goes through the +activator's quoting at render time, under the per-shell rules listed in T1. The [`pyvenv.cfg` writer][src-pyenv-cfg] +passes each key and value through the [line boundary helper][src-text]. virtualenv hashes each bundled wheel before +first use in each process, including when it reads the wheel from inside the zipapp. + +For economy of mechanism, one [helper][src-text] handles line breaks for both `pyvenv.cfg` and the batch activator. +Wheel integrity uses one comparison, a SHA-256 against the [embedded table][src-embed-init] for bundled wheels and +against PyPI's digest for downloaded ones. virtualenv implements no cryptography of its own. + +The design is open. We publish the code, the workflows and this document, and none of the defenses depends on an +attacker not knowing how they work. The [advisory list][advisories] holds each past advisory, and the +[release integrity][expl-release-integrity] explanation covers what the release checks prove. + +### Common weaknesses countered + +Each row pairs a weakness class relevant to virtualenv with the fix that counters it and the tests that fail if the fix +regresses. [CI][workflow-check] runs the unit tests, the [Hypothesis] property tests and the [Atheris] fuzz targets, as +the `property` and `fuzz` environments in [tox.toml][tox-toml] define them, on each pull request. + +| Weakness | Applies to | Fix | Tests | +| ------------------------------------------ | --------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [CWE-78] OS command injection | Destination path and `--prompt` in activators | Batch quoting, [#3250][pr-3250] ([GHSA-x78j-v8h9-3j2q][ghsa-x78j]), and refusing paths that quoting would alter, [#3280][pr-3280]. Path handling in bash and fish, [#3252][pr-3252] ([GHSA-p58f-9548-mpm2][ghsa-p58f]). csh history and prompt expansion, [#3256][pr-3256] and [#3263][pr-3263]. | [test_batch.py][tests-batch], [test_batch_quote.py][tests-batch-quote], [test_bash.py][tests-bash], [test_fish.py][tests-fish], [test_csh.py][tests-csh], [fuzz_powershell_quote.py][fuzz-powershell], [fuzz_nushell_quote.py][fuzz-nushell] | +| [CWE-93] CRLF injection | Keys and values in `pyvenv.cfg` | The writer turns line boundaries into spaces, [#3247][pr-3247] ([GHSA-9h9j-4vrj-gf7g][ghsa-9h9j]) | [test_pyenv_cfg.py][tests-pyenv-cfg], [fuzz_pyenv_cfg.py][fuzz-pyenv-cfg] | +| [CWE-88] argument injection | Distribution and version passed to `pip download` | virtualenv validates both before the call, [#3120][pr-3120] | [test_acquire.py][tests-acquire] | +| [CWE-22] path traversal | Wheel extraction into app-data, zipapp reads | virtualenv validates archive entries before extraction, [#3118][pr-3118], and confines [zipapp reads][src-zipapp] to the archive root, [#3121][pr-3121] | [test_bootstrap_link_via_app_data.py][tests-app-data], [test_util.py][tests-util] | +| [CWE-59] link following | Copying into an existing environment | virtualenv replaces a stale symlink at the destination instead of writing through it, [#3229][pr-3229] | [test_util.py][tests-util], [test_creator.py][tests-creator] | +| [CWE-367] TOCTOU race | App-data and [lock][src-lock] directory creation | Atomic directory creation in place of check-then-create, [#3013][pr-3013] ([GHSA-597g-3phw-6986][ghsa-597g], [CVE-2026-22702]) | No regression test came with the fix. The concurrency tests in [test_util.py][tests-util] and [test_bootstrap_link_via_app_data.py][tests-app-data] create directories in parallel. | +| [CWE-295] certificate validation | PyPI metadata request in the periodic update | No silent fallback to unverified TLS, [#3122][pr-3122] | [test_periodic_update.py][tests-periodic-update] | +| [CWE-494] download without integrity check | Bundled and downloaded seed wheels | virtualenv checks bundled wheels against the [embedded table][src-embed-init], [#3119][pr-3119], and downloaded wheels against PyPI's digest, [#3251][pr-3251] ([GHSA-94p9-xgh2-xp45][ghsa-94p9]), and refuses a downloaded wheel without one, [#3302][pr-3302] | [test_bundle.py][tests-bundle], [test_acquire.py][tests-acquire], [test_periodic_update.py][tests-periodic-update] | +| [CWE-732] incorrect permissions | Wheel image in app-data, deleting read-only files, executables in the environment | In symlink mode the wheel image is read-only. Deleting a read-only file adds owner write and keeps the other bits, [#3222][pr-3222]. [Marking a file executable][src-permission] adds execute bits and nothing else. | [test_util.py][tests-util] | + +Outside these rows, [pre-commit.ci][precommit-ci] runs [ruff] with each rule selected as a required check, which +includes the [flake8-bandit rules][ruff-bandit] apart from the subprocess and binding checks that +[pyproject.toml][pyproject] ignores. [CodeQL][workflow-codeql] scans the Python code and the workflows, [zizmor] audits +the workflows as I1 describes, and [scorecard.yaml][workflow-scorecard] runs [OpenSSF Scorecard][scorecard]. + +[advisories]: https://github.com/pypa/virtualenv/security/advisories +[api]: https://virtualenv.pypa.io/en/latest/reference/api.html +[atheris]: https://github.com/google/atheris +[bestpractices-virtualenv]: https://www.bestpractices.dev/projects/14689 +[bootstrap-12]: https://github.com/pypa/bootstrap/pull/12 +[ci-tools]: https://github.com/pypa/virtualenv/blob/main/tasks/ci-tools.json +[cli-app-data]: https://virtualenv.pypa.io/en/latest/reference/cli.html#app-data +[cli-download]: https://virtualenv.pypa.io/en/latest/reference/cli.html#download +[cli-extra-search-dir]: https://virtualenv.pypa.io/en/latest/reference/cli.html#extra-search-dir +[cli-no-periodic-update]: https://virtualenv.pypa.io/en/latest/reference/cli.html#no-periodic-update +[cli-prompt]: https://virtualenv.pypa.io/en/latest/reference/cli.html#prompt +[commons]: https://commons.apache.org/threat-model-0.1.0.html +[cve-2026-22702]: https://www.cve.org/CVERecord?id=CVE-2026-22702 +[cwe-22]: https://cwe.mitre.org/data/definitions/22.html +[cwe-295]: https://cwe.mitre.org/data/definitions/295.html +[cwe-367]: https://cwe.mitre.org/data/definitions/367.html +[cwe-494]: https://cwe.mitre.org/data/definitions/494.html +[cwe-59]: https://cwe.mitre.org/data/definitions/59.html +[cwe-732]: https://cwe.mitre.org/data/definitions/732.html +[cwe-78]: https://cwe.mitre.org/data/definitions/78.html +[cwe-88]: https://cwe.mitre.org/data/definitions/88.html +[cwe-93]: https://cwe.mitre.org/data/definitions/93.html +[cyclonedx]: https://cyclonedx.org/specification/overview/ +[dependabot-config]: https://github.com/pypa/virtualenv/blob/main/.github/dependabot.yaml +[dependabot-cooldown]: https://docs.github.com/en/code-security/dependabot/working-with-dependabot/dependabot-options-reference#cooldown- +[dev-ai]: https://virtualenv.pypa.io/en/latest/development.html#ai-assisted-contributions +[dev-maintainers]: https://virtualenv.pypa.io/en/latest/development.html#current-maintainers +[dev-recovery]: https://virtualenv.pypa.io/en/latest/development.html#recovering-a-partial-publication +[env-layout]: https://virtualenv.pypa.io/en/latest/reference/environment-layout.html +[expl-activators]: https://virtualenv.pypa.io/en/latest/explanation.html#activators +[expl-creators]: https://virtualenv.pypa.io/en/latest/explanation.html#creators +[expl-discovery]: https://virtualenv.pypa.io/en/latest/explanation.html#python-discovery +[expl-how]: https://virtualenv.pypa.io/en/latest/explanation.html#how-virtualenv-works +[expl-periodic]: https://virtualenv.pypa.io/en/latest/explanation.html#periodic-update-mechanism +[expl-release-integrity]: https://virtualenv.pypa.io/en/latest/explanation.html#release-integrity +[expl-seeders]: https://virtualenv.pypa.io/en/latest/explanation.html#seeders +[fuzz-nushell]: https://github.com/pypa/virtualenv/blob/main/tasks/fuzz_nushell_quote.py +[fuzz-powershell]: https://github.com/pypa/virtualenv/blob/main/tasks/fuzz_powershell_quote.py +[fuzz-pyenv-cfg]: https://github.com/pypa/virtualenv/blob/main/tasks/fuzz_pyenv_cfg.py +[get-virtualenv]: https://github.com/pypa/get-virtualenv +[gh-attestations]: https://docs.github.com/en/actions/security-for-github-actions/using-artifact-attestations/using-artifact-attestations-to-establish-provenance-for-builds +[gh-environments]: https://docs.github.com/en/actions/managing-workflow-runs-and-deployments/managing-deployments/managing-environments-for-deployment +[gh-immutable]: https://docs.github.com/en/code-security/supply-chain-security/understanding-your-software-supply-chain/immutable-releases +[gh-prt]: https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows#pull_request_target +[gh-push-protection]: https://docs.github.com/en/code-security/secret-scanning/introduction/about-push-protection +[gh-pvr]: https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing-information-about-vulnerabilities/privately-reporting-a-security-vulnerability +[gh-releases]: https://github.com/pypa/virtualenv/releases +[gh-rulesets]: https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/managing-rulesets/about-rulesets +[gh-secret-scanning]: https://docs.github.com/en/code-security/secret-scanning/introduction/about-secret-scanning +[gh-sha-pinning]: https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/enabling-features-for-your-repository/managing-github-actions-settings-for-a-repository#allowing-select-actions-and-reusable-workflows-to-run +[gh-token-permissions]: https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/enabling-features-for-your-repository/managing-github-actions-settings-for-a-repository#setting-the-permissions-of-the-github_token-for-your-repository +[ghsa-597g]: https://github.com/pypa/virtualenv/security/advisories/GHSA-597g-3phw-6986 +[ghsa-94p9]: https://github.com/pypa/virtualenv/security/advisories/GHSA-94p9-xgh2-xp45 +[ghsa-9h9j]: https://github.com/pypa/virtualenv/security/advisories/GHSA-9h9j-4vrj-gf7g +[ghsa-p58f]: https://github.com/pypa/virtualenv/security/advisories/GHSA-p58f-9548-mpm2 +[ghsa-x78j]: https://github.com/pypa/virtualenv/security/advisories/GHSA-x78j-v8h9-3j2q +[harden-runner]: https://github.com/step-security/harden-runner +[hatch]: https://hatch.pypa.io/ +[hatch-build]: https://github.com/pypa/virtualenv/blob/main/hatch_build.py +[hypothesis]: https://hypothesis.readthedocs.io/ +[ir]: https://github.com/pypa/virtualenv/blob/main/.github/INCIDENT_RESPONSE.md +[ir-after]: https://github.com/pypa/virtualenv/blob/main/.github/INCIDENT_RESPONSE.md#after-the-incident +[ir-response]: https://github.com/pypa/virtualenv/blob/main/.github/INCIDENT_RESPONSE.md#response +[ir-severity]: https://github.com/pypa/virtualenv/blob/main/.github/INCIDENT_RESPONSE.md#severity +[owasp-tm]: https://owasp.org/www-community/Threat_Modeling +[pep-503]: https://peps.python.org/pep-0503/ +[pep-751]: https://peps.python.org/pep-0751/ +[pep-770]: https://peps.python.org/pep-0770/ +[pip-config]: https://pip.pypa.io/en/stable/topics/configuration/ +[platformdirs]: https://platformdirs.readthedocs.io/ +[plugin-docs]: https://virtualenv.pypa.io/en/latest/plugin/index.html +[pr-3013]: https://github.com/pypa/virtualenv/pull/3013 +[pr-3118]: https://github.com/pypa/virtualenv/pull/3118 +[pr-3119]: https://github.com/pypa/virtualenv/pull/3119 +[pr-3120]: https://github.com/pypa/virtualenv/pull/3120 +[pr-3121]: https://github.com/pypa/virtualenv/pull/3121 +[pr-3122]: https://github.com/pypa/virtualenv/pull/3122 +[pr-3222]: https://github.com/pypa/virtualenv/pull/3222 +[pr-3229]: https://github.com/pypa/virtualenv/pull/3229 +[pr-3247]: https://github.com/pypa/virtualenv/pull/3247 +[pr-3250]: https://github.com/pypa/virtualenv/pull/3250 +[pr-3251]: https://github.com/pypa/virtualenv/pull/3251 +[pr-3252]: https://github.com/pypa/virtualenv/pull/3252 +[pr-3256]: https://github.com/pypa/virtualenv/pull/3256 +[pr-3263]: https://github.com/pypa/virtualenv/pull/3263 +[pr-3280]: https://github.com/pypa/virtualenv/pull/3280 +[pr-3293]: https://github.com/pypa/virtualenv/pull/3293 +[pr-3299]: https://github.com/pypa/virtualenv/pull/3299 +[pr-3302]: https://github.com/pypa/virtualenv/pull/3302 +[pr-3305]: https://github.com/pypa/virtualenv/pull/3305 +[pr-3306]: https://github.com/pypa/virtualenv/pull/3306 +[pr-3309]: https://github.com/pypa/virtualenv/pull/3309 +[pr-3310]: https://github.com/pypa/virtualenv/pull/3310 +[pr-3311]: https://github.com/pypa/virtualenv/pull/3311 +[precommit-ci]: https://pre-commit.ci/ +[precommit-config]: https://github.com/pypa/virtualenv/blob/main/.pre-commit-config.yaml +[pylock-zipapp]: https://github.com/pypa/virtualenv/blob/main/pylock.zipapp.toml +[pypa-bootstrap]: https://github.com/pypa/bootstrap +[pypi-2fa]: https://blog.pypi.org/posts/2024-01-01-2fa-enforced/ +[pypi-attestations]: https://docs.pypi.org/attestations/ +[pypi-json]: https://docs.pypi.org/api/json/ +[pypi-project]: https://pypi.org/project/virtualenv/ +[pypi-security]: https://pypi.org/security/ +[pypi-tp]: https://docs.pypi.org/trusted-publishers/ +[pyproject]: https://github.com/pypa/virtualenv/blob/main/pyproject.toml +[ra-files]: https://virtualenv.pypa.io/en/latest/reference/release-artifacts.html#files +[ra-repro]: https://virtualenv.pypa.io/en/latest/reference/release-artifacts.html#build-reproducibility +[release-21.10.0]: https://github.com/pypa/virtualenv/releases/tag/21.10.0 +[release-app]: https://github.com/apps/virtualenv-release +[release-lock]: https://github.com/pypa/virtualenv/blob/main/tasks/release-requirements.txt +[ruff]: https://docs.astral.sh/ruff/ +[ruff-bandit]: https://docs.astral.sh/ruff/rules/#flake8-bandit-s +[scorecard]: https://scorecard.dev/ +[scorecard-virtualenv]: https://scorecard.dev/viewer/?uri=github.com/pypa/virtualenv +[sde]: https://reproducible-builds.org/docs/source-date-epoch/ +[security]: https://github.com/pypa/virtualenv/blob/main/.github/SECURITY.md +[security-cra]: https://github.com/pypa/virtualenv/blob/main/.github/SECURITY.md#eu-cyber-resilience-act +[security-report]: https://github.com/pypa/virtualenv/blob/main/.github/SECURITY.md#reporting-a-vulnerability +[security-scope]: https://github.com/pypa/virtualenv/blob/main/.github/SECURITY.md#scope +[shiro]: https://shiro.apache.org/security-model.html +[slsa-provenance]: https://slsa.dev/spec/v1.0/provenance +[sosf-template]: https://gist.github.com/computersarebad/05d5ccbf13250489ea55a240b64c08d0 +[spdx]: https://spdx.github.io/spdx-spec/v2.3/ +[src-activation]: https://github.com/pypa/virtualenv/tree/main/src/virtualenv/activation +[src-app-data]: https://github.com/pypa/virtualenv/tree/main/src/virtualenv/app_data +[src-batch]: https://github.com/pypa/virtualenv/tree/main/src/virtualenv/activation/batch +[src-creator]: https://github.com/pypa/virtualenv/blob/main/src/virtualenv/create/creator.py +[src-embed-dir]: https://github.com/pypa/virtualenv/tree/main/src/virtualenv/seed/wheels/embed +[src-embed-init]: https://github.com/pypa/virtualenv/blob/main/src/virtualenv/seed/wheels/embed/__init__.py +[src-lock]: https://github.com/pypa/virtualenv/blob/main/src/virtualenv/util/lock.py +[src-periodic-update]: https://github.com/pypa/virtualenv/blob/main/src/virtualenv/seed/wheels/periodic_update.py +[src-permission]: https://github.com/pypa/virtualenv/blob/main/src/virtualenv/util/path/_permission.py +[src-plugin-base]: https://github.com/pypa/virtualenv/blob/main/src/virtualenv/run/plugin/base.py +[src-pyenv-cfg]: https://github.com/pypa/virtualenv/blob/main/src/virtualenv/create/pyenv_cfg.py +[src-symlink]: https://github.com/pypa/virtualenv/blob/main/src/virtualenv/seed/embed/via_app_data/pip_install/symlink.py +[src-text]: https://github.com/pypa/virtualenv/blob/main/src/virtualenv/util/text.py +[src-zipapp]: https://github.com/pypa/virtualenv/blob/main/src/virtualenv/util/zipapp.py +[stride]: https://learn.microsoft.com/en-us/azure/security/develop/threat-modeling-tool-threats#stride-model +[task-wheel-age]: https://github.com/pypa/virtualenv/blob/main/tasks/check_wheel_age.py +[tests-acquire]: https://github.com/pypa/virtualenv/blob/main/tests/unit/seed/wheels/test_acquire.py +[tests-app-data]: https://github.com/pypa/virtualenv/blob/main/tests/unit/seed/embed/test_bootstrap_link_via_app_data.py +[tests-bash]: https://github.com/pypa/virtualenv/blob/main/tests/unit/activation/test_bash.py +[tests-batch]: https://github.com/pypa/virtualenv/blob/main/tests/unit/activation/test_batch.py +[tests-batch-quote]: https://github.com/pypa/virtualenv/blob/main/tests/property/test_batch_quote.py +[tests-bundle]: https://github.com/pypa/virtualenv/blob/main/tests/unit/seed/wheels/test_bundle.py +[tests-creator]: https://github.com/pypa/virtualenv/blob/main/tests/unit/create/test_creator.py +[tests-csh]: https://github.com/pypa/virtualenv/blob/main/tests/unit/activation/test_csh.py +[tests-fish]: https://github.com/pypa/virtualenv/blob/main/tests/unit/activation/test_fish.py +[tests-periodic-update]: https://github.com/pypa/virtualenv/blob/main/tests/unit/seed/wheels/test_periodic_update.py +[tests-property]: https://github.com/pypa/virtualenv/tree/main/tests/property +[tests-pyenv-cfg]: https://github.com/pypa/virtualenv/blob/main/tests/property/test_pyenv_cfg.py +[tests-util]: https://github.com/pypa/virtualenv/blob/main/tests/unit/test_util.py +[tox]: https://tox.wiki/ +[tox-toml]: https://github.com/pypa/virtualenv/blob/main/tox.toml +[usage-config-location]: https://virtualenv.pypa.io/en/latest/how-to/usage.html#configuration-file-location +[usage-env]: https://virtualenv.pypa.io/en/latest/how-to/usage.html#environment-variables +[usage-insecure]: https://virtualenv.pypa.io/en/latest/how-to/usage.html#allow-unverified-https-for-periodic-updates +[usage-override-app-data]: https://virtualenv.pypa.io/en/latest/how-to/usage.html#override-app-data-location +[verify]: https://virtualenv.pypa.io/en/latest/how-to/verify-release.html +[verify-rebuild]: https://virtualenv.pypa.io/en/latest/how-to/verify-release.html#rebuild-the-release-files +[workflow-check]: https://github.com/pypa/virtualenv/blob/main/.github/workflows/check.yaml +[workflow-codeql]: https://github.com/pypa/virtualenv/blob/main/.github/workflows/codeql.yaml +[workflow-pre-release]: https://github.com/pypa/virtualenv/blob/main/.github/workflows/pre-release.yaml +[workflow-release]: https://github.com/pypa/virtualenv/blob/main/.github/workflows/release.yaml +[workflow-scorecard]: https://github.com/pypa/virtualenv/blob/main/.github/workflows/scorecard.yaml +[workflow-upgrade]: https://github.com/pypa/virtualenv/blob/main/.github/workflows/upgrade.yaml +[xz]: https://www.openwall.com/lists/oss-security/2024/03/29/4 +[zizmor]: https://docs.zizmor.sh/ diff --git a/.github/VULNERABILITY_REPORT.yml b/.github/VULNERABILITY_REPORT.yml new file mode 100644 index 000000000..0917872bd --- /dev/null +++ b/.github/VULNERABILITY_REPORT.yml @@ -0,0 +1,81 @@ +name: Report a security vulnerability +description: Privately report a vulnerability in virtualenv to its maintainers. +body: + - type: markdown + attributes: + value: | + Read the [security policy](https://github.com/pypa/virtualenv/security/policy) before you report: it lists what + is in and out of scope. Only the latest release is supported, so confirm the problem reproduces there. Write the + report yourself; reports that consist of unreviewed tool or model output take longer to triage. + - type: textarea + id: summary + attributes: + label: Summary + description: What the problem is and what an attacker gains from it. + validations: + required: true + - type: dropdown + id: area + attributes: + label: Affected area + description: The in-scope area from the security policy that the problem falls under. + options: + - Activation scripts (activate, activate.bat, activate.csh, activate.fish, activate.nu, Activate.ps1, activate.xsh) + - pyvenv.cfg or another file virtualenv writes + - Integrity of downloads (seed wheels, app-data cache) + - Writing outside the target directory (symlink, race, path handling) + - File or directory permissions + - Other + validations: + required: true + - type: textarea + id: input + attributes: + label: Attacker-controlled input + description: | + The value the attacker controls, such as the destination path, `--prompt`, project metadata or a downloaded file, + and how it reaches virtualenv. + validations: + required: true + - type: textarea + id: details + attributes: + label: Details + description: The code involved, with links to the source of the latest release. + validations: + required: true + - type: textarea + id: proof-of-concept + attributes: + label: Proof of concept + description: Steps or a script that reproduce the problem on the latest release. + render: shell + validations: + required: true + - type: textarea + id: impact + attributes: + label: Impact + description: Who is affected and what an attacker can do to them. + validations: + required: true + - type: input + id: versions + attributes: + label: Versions tested + description: virtualenv, Python, operating system and, for activation scripts, the shell. + placeholder: virtualenv 21.14.2, CPython 3.13.7, macOS 26.1, fish 4.9.3 + validations: + required: true + - type: textarea + id: mitigation + attributes: + label: Known mitigation + description: A workaround or fix you already know of, if any. + - type: checkboxes + id: latest-release + attributes: + label: Latest release + options: + - label: The problem reproduces on the latest virtualenv release. + required: true diff --git a/.github/codeql/codeql-config.yml b/.github/codeql/codeql-config.yml new file mode 100644 index 000000000..d6dbcb99f --- /dev/null +++ b/.github/codeql/codeql-config.yml @@ -0,0 +1,6 @@ +queries: + - uses: security-extended +threat-models: local +paths-ignore: + - .github/codeql/tests + - tests diff --git a/.github/codeql/codeql-workspace.yml b/.github/codeql/codeql-workspace.yml new file mode 100644 index 000000000..2ead4a38d --- /dev/null +++ b/.github/codeql/codeql-workspace.yml @@ -0,0 +1,2 @@ +provide: + - "*/qlpack.yml" diff --git a/.github/codeql/queries/MultilinePyvenvCfgValue.ql b/.github/codeql/queries/MultilinePyvenvCfgValue.ql new file mode 100644 index 000000000..4b2d72f25 --- /dev/null +++ b/.github/codeql/queries/MultilinePyvenvCfgValue.ql @@ -0,0 +1,47 @@ +/** + * @name Line boundary in a pyvenv.cfg value + * @description A pyvenv.cfg key or value reaches the file without collapse_line_boundaries(), so a prompt or path that + * carries a line boundary adds its own configuration lines when Python reads the file back. + * @kind path-problem + * @problem.severity error + * @security-severity 7.3 + * @precision high + * @id virtualenv/multiline-pyvenv-cfg-value + * @tags security + * external/cwe/cwe-93 + */ + +import python +import semmle.python.dataflow.new.DataFlow +import semmle.python.dataflow.new.TaintTracking + +predicate inPyEnvCfg(DataFlow::Node node) { + node.getScope().getScope*().(Class).getName().matches("%PyEnvCfg") +} + +module MultilinePyvenvCfgValueConfig implements DataFlow::ConfigSig { + predicate isSource(DataFlow::Node node) { + inPyEnvCfg(node) and node.(DataFlow::AttrRead).getAttributeName() = "content" + } + + predicate isSink(DataFlow::Node node) { + exists(DataFlow::MethodCallNode call | + inPyEnvCfg(call) and call.calls(_, "write_text") and node = call.getArg(0) + ) + } + + predicate isBarrier(DataFlow::Node node) { + exists(DataFlow::CallCfgNode call | + call.getFunction().asExpr().(Name).getId() = "collapse_line_boundaries" and node = call + ) + } +} + +module MultilinePyvenvCfgValueFlow = TaintTracking::Global; + +import MultilinePyvenvCfgValueFlow::PathGraph + +from MultilinePyvenvCfgValueFlow::PathNode source, MultilinePyvenvCfgValueFlow::PathNode sink +where MultilinePyvenvCfgValueFlow::flowPath(source, sink) +select sink.getNode(), source, sink, "$@ reaches pyvenv.cfg without collapsing line boundaries.", + source.getNode(), "This configuration value" diff --git a/.github/codeql/queries/UnquotedActivationValue.ql b/.github/codeql/queries/UnquotedActivationValue.ql new file mode 100644 index 000000000..e516bdc5b --- /dev/null +++ b/.github/codeql/queries/UnquotedActivationValue.ql @@ -0,0 +1,48 @@ +/** + * @name Unquoted value in an activation script + * @description A value from an activator's replacements() reaches an activation script template without going through + * the activator's quote(), so a crafted prompt or path can run shell commands on activation. + * @kind path-problem + * @problem.severity error + * @security-severity 7.8 + * @precision high + * @id virtualenv/unquoted-activation-value + * @tags security + * external/cwe/cwe-078 + */ + +import python +import semmle.python.dataflow.new.DataFlow +import semmle.python.dataflow.new.TaintTracking + +module UnquotedActivationValueConfig implements DataFlow::ConfigSig { + predicate isSource(DataFlow::Node node) { + exists(Function replacements, Return ret, Dict dict | + replacements.getName() = "replacements" and + ret.getScope() = replacements and + ret.getValue() = dict and + node.asExpr() = dict.getAValue() + ) + } + + predicate isSink(DataFlow::Node node) { + exists(DataFlow::MethodCallNode call | + call.calls(_, "replace") and + call.getScope().(Function).getName() = "instantiate_template" and + node = call.getArg(1) + ) + } + + predicate isBarrier(DataFlow::Node node) { + exists(DataFlow::MethodCallNode call | call.calls(_, "quote") and node = call) + } +} + +module UnquotedActivationValueFlow = TaintTracking::Global; + +import UnquotedActivationValueFlow::PathGraph + +from UnquotedActivationValueFlow::PathNode source, UnquotedActivationValueFlow::PathNode sink +where UnquotedActivationValueFlow::flowPath(source, sink) +select sink.getNode(), source, sink, "$@ reaches an activation script without quoting.", source.getNode(), + "This input" diff --git a/.github/codeql/queries/UnverifiedWheelDownload.ql b/.github/codeql/queries/UnverifiedWheelDownload.ql new file mode 100644 index 000000000..ddc994eb0 --- /dev/null +++ b/.github/codeql/queries/UnverifiedWheelDownload.ql @@ -0,0 +1,24 @@ +/** + * @name Downloaded wheel used without a digest check + * @description A function runs pip download without calling verify_wheel_digest(), so a compromised index or mirror can + * hand virtualenv a wheel that it then caches and installs into new environments. + * @kind problem + * @problem.severity error + * @security-severity 7.5 + * @precision high + * @id virtualenv/unverified-wheel-download + * @tags security + * external/cwe/cwe-494 + */ + +import python + +from Function download, Call popen +where + popen.getScope() = download and + popen.getFunc().(Name).getId() = "Popen" and + exists(StringLiteral subcommand | subcommand.getScope() = download and subcommand.getText() = "download") and + not exists(Call verify | + verify.getScope() = download and verify.getFunc().(Name).getId() = "verify_wheel_digest" + ) +select popen, "This pip download in $@ never calls verify_wheel_digest().", download, download.getName() diff --git a/.github/codeql/queries/qlpack.yml b/.github/codeql/queries/qlpack.yml new file mode 100644 index 000000000..8e1a8cdad --- /dev/null +++ b/.github/codeql/queries/qlpack.yml @@ -0,0 +1,4 @@ +name: pypa/virtualenv-queries +version: 0.0.1 +dependencies: + codeql/python-all: "*" diff --git a/.github/codeql/tests/MultilinePyvenvCfgValue/MultilinePyvenvCfgValue.expected b/.github/codeql/tests/MultilinePyvenvCfgValue/MultilinePyvenvCfgValue.expected new file mode 100644 index 000000000..f3e16ff65 --- /dev/null +++ b/.github/codeql/tests/MultilinePyvenvCfgValue/MultilinePyvenvCfgValue.expected @@ -0,0 +1,57 @@ +edges +| pyenv_cfg.py:19:9:19:12 | ControlFlowNode for text | pyenv_cfg.py:20:30:20:33 | ControlFlowNode for text | provenance | | +| pyenv_cfg.py:19:16:19:83 | ControlFlowNode for Attribute() | pyenv_cfg.py:19:9:19:12 | ControlFlowNode for text | provenance | | +| pyenv_cfg.py:19:24:19:43 | ControlFlowNode for Fstring | pyenv_cfg.py:19:24:19:43 | ControlFlowNode for Yield [List element] | provenance | | +| pyenv_cfg.py:19:24:19:43 | ControlFlowNode for Fstring | pyenv_cfg.py:19:24:19:43 | ControlFlowNode for Yield [List element] | provenance | | +| pyenv_cfg.py:19:24:19:82 | ControlFlowNode for .0 | pyenv_cfg.py:19:49:19:51 | ControlFlowNode for key | provenance | | +| pyenv_cfg.py:19:24:19:82 | ControlFlowNode for .0 | pyenv_cfg.py:19:49:19:58 | ControlFlowNode for Tuple | provenance | | +| pyenv_cfg.py:19:24:19:82 | ControlFlowNode for .0 | pyenv_cfg.py:19:49:19:58 | IterableElement | provenance | | +| pyenv_cfg.py:19:24:19:82 | ControlFlowNode for .0 | pyenv_cfg.py:19:54:19:58 | ControlFlowNode for value | provenance | | +| pyenv_cfg.py:19:24:19:82 | ControlFlowNode for .0 [List element, Tuple element at index 1] | pyenv_cfg.py:19:49:19:58 | ControlFlowNode for Tuple [Tuple element at index 1] | provenance | | +| pyenv_cfg.py:19:24:19:82 | ControlFlowNode for .0 [List element, Tuple element at index 1] | pyenv_cfg.py:19:49:19:58 | IterableSequence [Tuple element at index 1] | provenance | | +| pyenv_cfg.py:19:24:19:82 | ControlFlowNode for GeneratorExp [List element] | pyenv_cfg.py:19:16:19:83 | ControlFlowNode for Attribute() | provenance | str.join | +| pyenv_cfg.py:19:49:19:51 | ControlFlowNode for key | pyenv_cfg.py:19:24:19:43 | ControlFlowNode for Fstring | provenance | | +| pyenv_cfg.py:19:49:19:58 | ControlFlowNode for Tuple | pyenv_cfg.py:19:49:19:51 | ControlFlowNode for key | provenance | | +| pyenv_cfg.py:19:49:19:58 | ControlFlowNode for Tuple | pyenv_cfg.py:19:54:19:58 | ControlFlowNode for value | provenance | | +| pyenv_cfg.py:19:49:19:58 | ControlFlowNode for Tuple [Tuple element at index 0] | pyenv_cfg.py:19:49:19:51 | ControlFlowNode for key | provenance | | +| pyenv_cfg.py:19:49:19:58 | ControlFlowNode for Tuple [Tuple element at index 1] | pyenv_cfg.py:19:54:19:58 | ControlFlowNode for value | provenance | | +| pyenv_cfg.py:19:49:19:58 | ControlFlowNode for Tuple [Tuple element at index 1] | pyenv_cfg.py:19:54:19:58 | ControlFlowNode for value | provenance | | +| pyenv_cfg.py:19:49:19:58 | IterableElement | pyenv_cfg.py:19:49:19:58 | ControlFlowNode for Tuple [Tuple element at index 0] | provenance | | +| pyenv_cfg.py:19:49:19:58 | IterableElement | pyenv_cfg.py:19:49:19:58 | ControlFlowNode for Tuple [Tuple element at index 1] | provenance | | +| pyenv_cfg.py:19:49:19:58 | IterableSequence [Tuple element at index 1] | pyenv_cfg.py:19:49:19:58 | ControlFlowNode for Tuple [Tuple element at index 1] | provenance | | +| pyenv_cfg.py:19:54:19:58 | ControlFlowNode for value | pyenv_cfg.py:19:24:19:43 | ControlFlowNode for Fstring | provenance | | +| pyenv_cfg.py:19:54:19:58 | ControlFlowNode for value | pyenv_cfg.py:19:24:19:43 | ControlFlowNode for Fstring | provenance | | +| pyenv_cfg.py:19:63:19:74 | ControlFlowNode for Attribute | pyenv_cfg.py:19:63:19:82 | ControlFlowNode for Attribute() | provenance | dict.items | +| pyenv_cfg.py:19:63:19:74 | ControlFlowNode for Attribute | pyenv_cfg.py:19:63:19:82 | ControlFlowNode for Attribute() [List element, Tuple element at index 1] | provenance | dict.items | +| pyenv_cfg.py:19:63:19:82 | ControlFlowNode for Attribute() | pyenv_cfg.py:19:24:19:82 | ControlFlowNode for .0 | provenance | | +| pyenv_cfg.py:19:63:19:82 | ControlFlowNode for Attribute() | pyenv_cfg.py:19:24:19:82 | ControlFlowNode for GeneratorExp [List element] | provenance | | +| pyenv_cfg.py:19:63:19:82 | ControlFlowNode for Attribute() [List element, Tuple element at index 1] | pyenv_cfg.py:19:24:19:82 | ControlFlowNode for .0 [List element, Tuple element at index 1] | provenance | | +| pyenv_cfg.py:19:63:19:82 | ControlFlowNode for Attribute() [List element, Tuple element at index 1] | pyenv_cfg.py:19:24:19:82 | ControlFlowNode for GeneratorExp [List element] | provenance | | +nodes +| pyenv_cfg.py:19:9:19:12 | ControlFlowNode for text | semmle.label | ControlFlowNode for text | +| pyenv_cfg.py:19:16:19:83 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() | +| pyenv_cfg.py:19:24:19:43 | ControlFlowNode for Fstring | semmle.label | ControlFlowNode for Fstring | +| pyenv_cfg.py:19:24:19:43 | ControlFlowNode for Fstring | semmle.label | ControlFlowNode for Fstring | +| pyenv_cfg.py:19:24:19:43 | ControlFlowNode for Yield [List element] | semmle.label | ControlFlowNode for Yield [List element] | +| pyenv_cfg.py:19:24:19:43 | ControlFlowNode for Yield [List element] | semmle.label | ControlFlowNode for Yield [List element] | +| pyenv_cfg.py:19:24:19:82 | ControlFlowNode for .0 | semmle.label | ControlFlowNode for .0 | +| pyenv_cfg.py:19:24:19:82 | ControlFlowNode for .0 [List element, Tuple element at index 1] | semmle.label | ControlFlowNode for .0 [List element, Tuple element at index 1] | +| pyenv_cfg.py:19:24:19:82 | ControlFlowNode for GeneratorExp [List element] | semmle.label | ControlFlowNode for GeneratorExp [List element] | +| pyenv_cfg.py:19:49:19:51 | ControlFlowNode for key | semmle.label | ControlFlowNode for key | +| pyenv_cfg.py:19:49:19:58 | ControlFlowNode for Tuple | semmle.label | ControlFlowNode for Tuple | +| pyenv_cfg.py:19:49:19:58 | ControlFlowNode for Tuple [Tuple element at index 0] | semmle.label | ControlFlowNode for Tuple [Tuple element at index 0] | +| pyenv_cfg.py:19:49:19:58 | ControlFlowNode for Tuple [Tuple element at index 1] | semmle.label | ControlFlowNode for Tuple [Tuple element at index 1] | +| pyenv_cfg.py:19:49:19:58 | ControlFlowNode for Tuple [Tuple element at index 1] | semmle.label | ControlFlowNode for Tuple [Tuple element at index 1] | +| pyenv_cfg.py:19:49:19:58 | IterableElement | semmle.label | IterableElement | +| pyenv_cfg.py:19:49:19:58 | IterableSequence [Tuple element at index 1] | semmle.label | IterableSequence [Tuple element at index 1] | +| pyenv_cfg.py:19:54:19:58 | ControlFlowNode for value | semmle.label | ControlFlowNode for value | +| pyenv_cfg.py:19:54:19:58 | ControlFlowNode for value | semmle.label | ControlFlowNode for value | +| pyenv_cfg.py:19:63:19:74 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute | +| pyenv_cfg.py:19:63:19:82 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() | +| pyenv_cfg.py:19:63:19:82 | ControlFlowNode for Attribute() [List element, Tuple element at index 1] | semmle.label | ControlFlowNode for Attribute() [List element, Tuple element at index 1] | +| pyenv_cfg.py:20:30:20:33 | ControlFlowNode for text | semmle.label | ControlFlowNode for text | +subpaths +| pyenv_cfg.py:19:63:19:82 | ControlFlowNode for Attribute() | pyenv_cfg.py:19:24:19:82 | ControlFlowNode for .0 | pyenv_cfg.py:19:24:19:43 | ControlFlowNode for Yield [List element] | pyenv_cfg.py:19:24:19:82 | ControlFlowNode for GeneratorExp [List element] | +| pyenv_cfg.py:19:63:19:82 | ControlFlowNode for Attribute() [List element, Tuple element at index 1] | pyenv_cfg.py:19:24:19:82 | ControlFlowNode for .0 [List element, Tuple element at index 1] | pyenv_cfg.py:19:24:19:43 | ControlFlowNode for Yield [List element] | pyenv_cfg.py:19:24:19:82 | ControlFlowNode for GeneratorExp [List element] | +#select +| pyenv_cfg.py:20:30:20:33 | ControlFlowNode for text | pyenv_cfg.py:19:63:19:74 | ControlFlowNode for Attribute | pyenv_cfg.py:20:30:20:33 | ControlFlowNode for text | $@ reaches pyvenv.cfg without collapsing line boundaries. | pyenv_cfg.py:19:63:19:74 | ControlFlowNode for Attribute | This configuration value | diff --git a/.github/codeql/tests/MultilinePyvenvCfgValue/MultilinePyvenvCfgValue.qlref b/.github/codeql/tests/MultilinePyvenvCfgValue/MultilinePyvenvCfgValue.qlref new file mode 100644 index 000000000..b97c0a129 --- /dev/null +++ b/.github/codeql/tests/MultilinePyvenvCfgValue/MultilinePyvenvCfgValue.qlref @@ -0,0 +1 @@ +MultilinePyvenvCfgValue.ql diff --git a/.github/codeql/tests/MultilinePyvenvCfgValue/pyenv_cfg.py b/.github/codeql/tests/MultilinePyvenvCfgValue/pyenv_cfg.py new file mode 100644 index 000000000..abd939e54 --- /dev/null +++ b/.github/codeql/tests/MultilinePyvenvCfgValue/pyenv_cfg.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from pathlib import Path + + +def collapse_line_boundaries(text: str) -> str: + return " ".join(text.splitlines()) + + +class RawPyEnvCfg: + def __init__(self, content: dict[str, str], path: Path) -> None: + self.content = content + self.path = path + + def write(self) -> None: + text = "".join(f"{key} = {value}\n" for key, value in self.content.items()) + self.path.write_text(text, encoding="utf-8") + + +class CollapsedPyEnvCfg: + def __init__(self, content: dict[str, str], path: Path) -> None: + self.content = content + self.path = path + + def write(self) -> None: + text = "".join( + f"{collapse_line_boundaries(key)} = {collapse_line_boundaries(value)}\n" + for key, value in self.content.items() + ) + self.path.write_text(text, encoding="utf-8") diff --git a/.github/codeql/tests/UnquotedActivationValue/UnquotedActivationValue.expected b/.github/codeql/tests/UnquotedActivationValue/UnquotedActivationValue.expected new file mode 100644 index 000000000..f463dd25d --- /dev/null +++ b/.github/codeql/tests/UnquotedActivationValue/UnquotedActivationValue.expected @@ -0,0 +1,28 @@ +edges +| activators.py:8:16:8:70 | ControlFlowNode for Dict [Dictionary element at key __VIRTUAL_ENV__] | activators.py:11:27:11:57 | ControlFlowNode for Attribute() [Dictionary element at key __VIRTUAL_ENV__] | provenance | | +| activators.py:8:16:8:70 | ControlFlowNode for Dict [Dictionary element at key __VIRTUAL_PROMPT__] | activators.py:11:27:11:57 | ControlFlowNode for Attribute() [Dictionary element at key __VIRTUAL_PROMPT__] | provenance | | +| activators.py:8:39:8:44 | ControlFlowNode for prompt | activators.py:8:16:8:70 | ControlFlowNode for Dict [Dictionary element at key __VIRTUAL_PROMPT__] | provenance | | +| activators.py:8:66:8:69 | ControlFlowNode for dest | activators.py:8:16:8:70 | ControlFlowNode for Dict [Dictionary element at key __VIRTUAL_ENV__] | provenance | | +| activators.py:11:13:11:22 | ControlFlowNode for Tuple [Tuple element at index 1] | activators.py:11:18:11:22 | ControlFlowNode for value | provenance | | +| activators.py:11:13:11:22 | IterableSequence [Tuple element at index 1] | activators.py:11:13:11:22 | ControlFlowNode for Tuple [Tuple element at index 1] | provenance | | +| activators.py:11:18:11:22 | ControlFlowNode for value | activators.py:12:38:12:42 | ControlFlowNode for value | provenance | | +| activators.py:11:27:11:57 | ControlFlowNode for Attribute() [Dictionary element at key __VIRTUAL_ENV__] | activators.py:11:27:11:65 | ControlFlowNode for Attribute() [List element, Tuple element at index 1] | provenance | dict.items | +| activators.py:11:27:11:57 | ControlFlowNode for Attribute() [Dictionary element at key __VIRTUAL_PROMPT__] | activators.py:11:27:11:65 | ControlFlowNode for Attribute() [List element, Tuple element at index 1] | provenance | dict.items | +| activators.py:11:27:11:65 | ControlFlowNode for Attribute() [List element, Tuple element at index 1] | activators.py:11:13:11:22 | ControlFlowNode for Tuple [Tuple element at index 1] | provenance | | +| activators.py:11:27:11:65 | ControlFlowNode for Attribute() [List element, Tuple element at index 1] | activators.py:11:13:11:22 | IterableSequence [Tuple element at index 1] | provenance | | +nodes +| activators.py:8:16:8:70 | ControlFlowNode for Dict [Dictionary element at key __VIRTUAL_ENV__] | semmle.label | ControlFlowNode for Dict [Dictionary element at key __VIRTUAL_ENV__] | +| activators.py:8:16:8:70 | ControlFlowNode for Dict [Dictionary element at key __VIRTUAL_PROMPT__] | semmle.label | ControlFlowNode for Dict [Dictionary element at key __VIRTUAL_PROMPT__] | +| activators.py:8:39:8:44 | ControlFlowNode for prompt | semmle.label | ControlFlowNode for prompt | +| activators.py:8:66:8:69 | ControlFlowNode for dest | semmle.label | ControlFlowNode for dest | +| activators.py:11:13:11:22 | ControlFlowNode for Tuple [Tuple element at index 1] | semmle.label | ControlFlowNode for Tuple [Tuple element at index 1] | +| activators.py:11:13:11:22 | IterableSequence [Tuple element at index 1] | semmle.label | IterableSequence [Tuple element at index 1] | +| activators.py:11:18:11:22 | ControlFlowNode for value | semmle.label | ControlFlowNode for value | +| activators.py:11:27:11:57 | ControlFlowNode for Attribute() [Dictionary element at key __VIRTUAL_ENV__] | semmle.label | ControlFlowNode for Attribute() [Dictionary element at key __VIRTUAL_ENV__] | +| activators.py:11:27:11:57 | ControlFlowNode for Attribute() [Dictionary element at key __VIRTUAL_PROMPT__] | semmle.label | ControlFlowNode for Attribute() [Dictionary element at key __VIRTUAL_PROMPT__] | +| activators.py:11:27:11:65 | ControlFlowNode for Attribute() [List element, Tuple element at index 1] | semmle.label | ControlFlowNode for Attribute() [List element, Tuple element at index 1] | +| activators.py:12:38:12:42 | ControlFlowNode for value | semmle.label | ControlFlowNode for value | +subpaths +#select +| activators.py:12:38:12:42 | ControlFlowNode for value | activators.py:8:39:8:44 | ControlFlowNode for prompt | activators.py:12:38:12:42 | ControlFlowNode for value | $@ reaches an activation script without quoting. | activators.py:8:39:8:44 | ControlFlowNode for prompt | This input | +| activators.py:12:38:12:42 | ControlFlowNode for value | activators.py:8:66:8:69 | ControlFlowNode for dest | activators.py:12:38:12:42 | ControlFlowNode for value | $@ reaches an activation script without quoting. | activators.py:8:66:8:69 | ControlFlowNode for dest | This input | diff --git a/.github/codeql/tests/UnquotedActivationValue/UnquotedActivationValue.qlref b/.github/codeql/tests/UnquotedActivationValue/UnquotedActivationValue.qlref new file mode 100644 index 000000000..92c6c0463 --- /dev/null +++ b/.github/codeql/tests/UnquotedActivationValue/UnquotedActivationValue.qlref @@ -0,0 +1 @@ +UnquotedActivationValue.ql diff --git a/.github/codeql/tests/UnquotedActivationValue/activators.py b/.github/codeql/tests/UnquotedActivationValue/activators.py new file mode 100644 index 000000000..de996e822 --- /dev/null +++ b/.github/codeql/tests/UnquotedActivationValue/activators.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +import shlex + + +class UnquotedActivator: + def replacements(self, prompt: str, dest: str) -> dict[str, str]: + return {"__VIRTUAL_PROMPT__": prompt, "__VIRTUAL_ENV__": dest} + + def instantiate_template(self, text: str, prompt: str, dest: str) -> str: + for key, value in self.replacements(prompt, dest).items(): + text = text.replace(key, value) + return text + + +class QuotedActivator: + @staticmethod + def quote(string: str) -> str: + return shlex.quote(string) + + def replacements(self, prompt: str, dest: str) -> dict[str, str]: + return {"__VIRTUAL_PROMPT__": prompt, "__VIRTUAL_ENV__": dest} + + def instantiate_template(self, text: str, prompt: str, dest: str) -> str: + for key, value in self.replacements(prompt, dest).items(): + text = text.replace(key, self.quote(value)) + return text diff --git a/.github/codeql/tests/UnverifiedWheelDownload/UnverifiedWheelDownload.expected b/.github/codeql/tests/UnverifiedWheelDownload/UnverifiedWheelDownload.expected new file mode 100644 index 000000000..fc2f06138 --- /dev/null +++ b/.github/codeql/tests/UnverifiedWheelDownload/UnverifiedWheelDownload.expected @@ -0,0 +1 @@ +| acquire.py:11:5:11:58 | Popen() | This pip download in $@ never calls verify_wheel_digest(). | acquire.py:10:1:10:47 | Function download_unverified | download_unverified | diff --git a/.github/codeql/tests/UnverifiedWheelDownload/UnverifiedWheelDownload.qlref b/.github/codeql/tests/UnverifiedWheelDownload/UnverifiedWheelDownload.qlref new file mode 100644 index 000000000..2170814a5 --- /dev/null +++ b/.github/codeql/tests/UnverifiedWheelDownload/UnverifiedWheelDownload.qlref @@ -0,0 +1 @@ +UnverifiedWheelDownload.ql diff --git a/.github/codeql/tests/UnverifiedWheelDownload/acquire.py b/.github/codeql/tests/UnverifiedWheelDownload/acquire.py new file mode 100644 index 000000000..4186bbf1b --- /dev/null +++ b/.github/codeql/tests/UnverifiedWheelDownload/acquire.py @@ -0,0 +1,18 @@ +from __future__ import annotations + +from subprocess import Popen + + +def verify_wheel_digest(wheel: str) -> None: + del wheel + + +def download_unverified(to_folder: str) -> str: + Popen(["pip", "download", "--dest", to_folder, "pip"]).wait() # ruff:ignore[start-process-with-partial-path] + return to_folder + + +def download_verified(to_folder: str) -> str: + Popen(["pip", "download", "--dest", to_folder, "pip"]).wait() # ruff:ignore[start-process-with-partial-path] + verify_wheel_digest(to_folder) + return to_folder diff --git a/.github/codeql/tests/qlpack.yml b/.github/codeql/tests/qlpack.yml new file mode 100644 index 000000000..3879fa271 --- /dev/null +++ b/.github/codeql/tests/qlpack.yml @@ -0,0 +1,6 @@ +name: pypa/virtualenv-queries-tests +extractor: python +tests: . +dependencies: + codeql/python-all: "*" + pypa/virtualenv-queries: "*" diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml new file mode 100644 index 000000000..acd659bbe --- /dev/null +++ b/.github/dependabot.yaml @@ -0,0 +1,24 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + day: "tuesday" + groups: + github-actions: + patterns: + - "*" + cooldown: + default-days: 7 + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "weekly" + day: "tuesday" + groups: + runtime-dependencies: + patterns: + - "*" + cooldown: + default-days: 7 diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index 123014908..000000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,6 +0,0 @@ -version: 2 -updates: - - package-ecosystem: "github-actions" - directory: "/" - schedule: - interval: "daily" diff --git a/.github/release.yml b/.github/release.yaml similarity index 100% rename from .github/release.yml rename to .github/release.yaml diff --git a/.github/workflows/check.yaml b/.github/workflows/check.yaml index 68120bde6..f0cdd1be9 100644 --- a/.github/workflows/check.yaml +++ b/.github/workflows/check.yaml @@ -6,21 +6,26 @@ on: pull_request: schedule: - cron: "0 8 * * *" - concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true - +permissions: {} jobs: test: - name: 🧪 test ${{ matrix.py }} - ${{ matrix.os }} + name: 🧪 test ${{ matrix.py }} - ${{ matrix.os }} ${{ matrix.group }} if: github.event_name != 'schedule' || github.repository_owner == 'pypa' runs-on: ${{ matrix.os }} - timeout-minutes: 40 + timeout-minutes: ${{ matrix.py == 'graalpy' && 20 || 40 }} + permissions: + contents: read + env: + PIP_DISABLE_PIP_VERSION_CHECK: "1" strategy: fail-fast: false matrix: py: + - "3.15t" + - "3.15" - "3.14t" - "3.14" - "3.13t" @@ -29,120 +34,339 @@ jobs: - "3.11" - "3.10" - "3.9" - - "3.8" - pypy-3.11 - pypy-3.10 - pypy-3.9 - - pypy-3.8 - - graalpy-24.1 + - rp os: - ubuntu-24.04 - - macos-15 - - windows-2025 + - macos-26 + - windows-2025-vs2026 include: - - { os: macos-15, py: "brew@3.11" } - - { os: macos-15, py: "brew@3.10" } - - { os: macos-15, py: "brew@3.9" } + - {os: windows-2025-vs2026, py: "3.13t", diagnostics: true} + - {os: macos-26, py: "3.13t", diagnostics: true} + - {os: ubuntu-24.04, py: graalpy, group: "1/4"} + - {os: ubuntu-24.04, py: graalpy, group: "2/4"} + - {os: ubuntu-24.04, py: graalpy, group: "3/4"} + - {os: ubuntu-24.04, py: graalpy, group: "4/4"} + - {os: macos-26, py: graalpy, group: "1/4"} + - {os: macos-26, py: graalpy, group: "2/4"} + - {os: macos-26, py: graalpy, group: "3/4"} + - {os: macos-26, py: graalpy, group: "4/4"} + - {os: macos-26, py: "brew@3.14"} + - {os: macos-26, py: "brew@3.13"} + - {os: macos-26, py: "brew@3.12"} + - {os: macos-26, py: "brew@3.11"} + - {os: macos-26, py: "brew@3.10"} exclude: - - { os: windows-2025, py: "graalpy-24.1" } - - { os: windows-2025, py: "pypy-3.10" } - - { os: windows-2025, py: "pypy-3.9" } - - { os: windows-2025, py: "pypy-3.8" } + - {os: windows-2025-vs2026, py: "pypy-3.10"} + - {os: windows-2025-vs2026, py: "pypy-3.9"} steps: - - name: 🚀 Install uv - uses: astral-sh/setup-uv@v4 - name: 📥 Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 - - name: 🐍 Setup Python for tox - uses: actions/setup-python@v5 + persist-credentials: false + - name: 🏷️ Fetch upstream tags for versioning + shell: bash + run: | + git fetch --force --tags https://github.com/pypa/virtualenv.git + - name: 🚀 Install uv + uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 with: - python-version: "3.14" + enable-cache: true + cache-dependency-glob: "pyproject.toml" + cache-suffix: ${{ matrix.py }} + python-version: "3.15" + activate-environment: true - name: 📦 Install tox with this virtualenv shell: bash + env: + MATRIX_PY: ${{ matrix.py }} run: | - if [[ "${{ matrix.py }}" == "3.13t" || "${{ matrix.py }}" == "3.14t" ]]; then - uv tool install --no-managed-python --python 3.14 tox --with . + if [[ "$MATRIX_PY" == "3.13t" || "$MATRIX_PY" == "3.14t" || "$MATRIX_PY" == "3.15t" || "$MATRIX_PY" == "rp" ]]; then + uv tool install --python-preference only-managed --python 3.15 "tox>=4.45" --with . else - uv tool install --no-managed-python --python 3.14 tox --with tox-uv --with . + uv tool install --python-preference only-managed --python 3.15 "tox>=4.45" --with tox-uv --with . fi + - name: 🧮 Resolve GraalPy release + id: graalpy + if: matrix.py == 'graalpy' + shell: bash + # the upgrade workflow keeps tasks/ci-tools.json on the newest release + run: echo "python-version=graalpy-$(jq --raw-output '.graalpy.tag | ltrimstr("graal-")' tasks/ci-tools.json)" >> "$GITHUB_OUTPUT" - name: 🐍 Setup Python for test ${{ matrix.py }} - uses: actions/setup-python@v5 - if: ${{ !startsWith(matrix.py, 'brew@') }} + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + if: ${{ !startsWith(matrix.py, 'brew@') && !startsWith(matrix.py, '3.15') && matrix.py != 'rp' }} with: + python-version: ${{ steps.graalpy.outputs.python-version || matrix.py }} + allow-prereleases: true + - name: Setup Python 3.15 for tests + if: startsWith(matrix.py, '3.15') + uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 + with: + enable-cache: false python-version: ${{ matrix.py }} + activate-environment: true + - name: 🦀 Setup RustPython + if: matrix.py == 'rp' + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + # the upgrade workflow keeps tasks/ci-tools.json on the newest release; tr drops the CR jq.exe may emit + read -r tag asset sha256 < <(jq --raw-output --arg os "$RUNNER_OS" \ + '.rustpython | [.tag, .assets[$os].name, .assets[$os].sha256] | @tsv' tasks/ci-tools.json | tr -d '\r') + gh release download "$tag" --repo RustPython/RustPython --pattern "$asset" --dir "$RUNNER_TEMP" + (cd "$RUNNER_TEMP" && echo "$sha256 $asset" | sha256sum -c -) + if [ "$RUNNER_OS" = "Windows" ]; then + cp "$RUNNER_TEMP/$asset" "/c/ProgramData/chocolatey/bin/rustpython.exe" + else + chmod +x "$RUNNER_TEMP/$asset" + sudo mv "$RUNNER_TEMP/$asset" /usr/local/bin/rustpython + fi - name: 🛠️ Install OS dependencies shell: bash + env: + GH_TOKEN: ${{ github.token }} + MATRIX_PY: ${{ matrix.py }} run: | - if [ "${{ runner.os }}" = "Linux" ]; then - sudo apt-get install -y software-properties-common - sudo apt-add-repository ppa:fish-shell/release-4 -y - curl -fsSL https://apt.fury.io/nushell/gpg.key | sudo gpg --dearmor -o /etc/apt/trusted.gpg.d/fury-nushell.gpg - echo "deb https://apt.fury.io/nushell/ /" | sudo tee /etc/apt/sources.list.d/fury.list - sudo apt-get update -y - sudo apt-get install snapd fish csh nushell -y - elif [ "${{ runner.os }}" = "macOS" ]; then + if [ "$RUNNER_OS" = "Linux" ]; then + retry() { + local n=1 + while ! "$@"; do + if [ $n -ge 5 ]; then + echo "Command failed after $n attempts: $*" >&2 + return 1 + fi + echo "Attempt $n failed for: $*; retrying in 10s" >&2 + n=$((n + 1)) + sleep 10 + done + } + retry sudo apt-get install -y software-properties-common + retry sudo apt-add-repository ppa:fish-shell/release-4 -y + sudo install -m 0755 -d /etc/apt/keyrings + curl -fsSL --retry 5 --retry-all-errors --retry-delay 10 https://apt.fury.io/nushell/gpg.key \ + | sudo gpg --dearmor -o /etc/apt/keyrings/fury-nushell.gpg + sudo chmod 0644 /etc/apt/keyrings/fury-nushell.gpg + echo "deb [signed-by=/etc/apt/keyrings/fury-nushell.gpg] https://apt.fury.io/nushell/ /" \ + | sudo tee /etc/apt/sources.list.d/fury.list + retry sudo apt-get update -y + retry sudo apt-get install snapd fish csh nushell -y + elif [ "$RUNNER_OS" = "macOS" ]; then brew update - if [[ "${{ matrix.py }}" == brew@* ]]; then - PY=$(echo '${{ matrix.py }}' | cut -c 6-) - brew install python@$PY || brew upgrade python@$PY + if [[ "$MATRIX_PY" == brew@* ]]; then + PY=$(echo "$MATRIX_PY" | cut -c 6-) + brew list --versions "python@$PY" >/dev/null 2>&1 || brew install "python@$PY" echo "/usr/local/opt/python@$PY/libexec/bin" >>"${GITHUB_PATH}" fi brew install fish tcsh nushell || brew upgrade fish tcsh nushell - elif [ "${{ runner.os }}" = "Windows" ]; then - choco install nushell + elif [ "$RUNNER_OS" = "Windows" ]; then + read -r nu_tag nu_archive nu_sha256 < <(jq --raw-output \ + '.nushell | [.tag, .assets.Windows.name, .assets.Windows.sha256] | @tsv' tasks/ci-tools.json | tr -d '\r') + gh release download "$nu_tag" --repo nushell/nushell --pattern "$nu_archive" --dir "$RUNNER_TEMP" + (cd "$RUNNER_TEMP" && echo "$nu_sha256 $nu_archive" | sha256sum -c -) + 7z x "$RUNNER_TEMP/$nu_archive" -o"$RUNNER_TEMP/nushell" -y + cp "$RUNNER_TEMP/nushell/nu.exe" "/c/ProgramData/chocolatey/bin/nu.exe" fi - name: 🧬 Pick environment to run shell: bash + env: + MATRIX_PY: ${{ matrix.py }} run: | - py="${{ matrix.py }}" + py="$MATRIX_PY" if [[ "$py" == brew@* ]]; then brew_version="${py#brew@}" - echo "TOX_DISCOVER=/opt/homebrew/bin/python${brew_version}" >> "$GITHUB_ENV" + echo "UV_PYTHON=/opt/homebrew/bin/python${brew_version}" >> "$GITHUB_ENV" py="$brew_version" fi - [[ "$py" == graalpy-* ]] && py="graalpy" echo "TOXENV=$py" >> "$GITHUB_ENV" echo "Set TOXENV=$py" - name: 🏗️ Setup test suite run: tox run -vvvv --notest --skip-missing-interpreters false + - name: 🔎 Verify pytest diagnostics + if: matrix.diagnostics + shell: bash + run: | + if [ "$RUNNER_OS" = "Windows" ]; then + python_path=./.tox/3.13t/Scripts/python.exe + else + python_path=./.tox/3.13t/bin/python + fi + "$python_path" -m pytest tests/tasks/test_pytest_diagnostics.py -n 0 --basetemp=.tox/diagnostics-self-test - name: 🏃 Run test suite run: tox run --skip-pkg-install - timeout-minutes: 20 + timeout-minutes: ${{ matrix.py == 'graalpy' && 17 || 30 }} env: - PYTEST_ADDOPTS: "-vv --durations=20" + GRAALPY_GROUP: ${{ matrix.group || '1/1' }} + PYTEST_PLUGINS: ${{ matrix.diagnostics && 'tasks.pytest_diagnostics' || '' }} + # --reruns lets a single test recover from a transient worker crash (see + # pytest-rerunfailures' documented pytest-xdist crash-recovery support) instead of + # failing the whole matrix leg; scoped to CI only so local runs still fail fast. + # graalpy skips xdist: tasks/run_graalpy_tests.py runs one serial pytest process per + # CPU instead, each on its own --shard slice of the tests. graalpy also dropped its + # --timeout-method=thread override: that method spawns a watcher thread per test, + # and GraalVM's Context.close() treats any Python-visible thread still alive as + # fatal, crashing the job even after every test passed (see #3240). The default + # signal-based method needs no such thread. --timeout=300 stays as a safety net. + PYTEST_ADDOPTS: "-vv --durations=20 --reruns 2 --reruns-delay 2${{ matrix.py == 'graalpy' && ' --timeout=300' || '' }}" + # If a worker process dies outright (seen on 3.13t/Windows with no Python-level + # exception at all), this prints a native stack trace to stderr instead of nothing. + PYTHONFAULTHANDLER: "1" CI_RUN: "yes" DIFF_AGAINST: HEAD + - name: 📤 Store macOS crash reports + if: always() && matrix.diagnostics && runner.os == 'macOS' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: crash-reports-${{ matrix.py }}-${{ matrix.os }} + path: ~/Library/Logs/DiagnosticReports/ + if-no-files-found: ignore + retention-days: 14 + - name: 📤 Store pytest diagnostics + if: always() && matrix.diagnostics + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: hang-diagnostics-${{ matrix.py }}-${{ matrix.os }} + path: | + .tox/hang-diagnostics/ + .tox/diagnostics-self-test/**/diagnostics/*.log + .tox/3.13t/log/ + .tox/junit.3.13t.xml + include-hidden-files: true + if-no-files-found: ignore + retention-days: 14 + - name: 📤 Store coverage data + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: coverage-${{ matrix.py }}-${{ matrix.os }}-${{ strategy.job-index }} + # combined .coverage. files and the per-process files of envs that do not combine them + path: .tox/.coverage* + include-hidden-files: true + if-no-files-found: error + retention-days: 1 + coverage: + name: 📊 coverage + needs: test + if: github.event_name != 'schedule' || github.repository_owner == 'pypa' + runs-on: ubuntu-24.04 + permissions: + contents: read # check out the sources the report reads + steps: + - name: 📥 Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: 🚀 Install uv + uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 + with: + python-version: "3.15" + enable-cache: true + cache-dependency-glob: "pyproject.toml" + cache-suffix: coverage + - name: 📦 Install tox + run: uv tool install --python-preference only-managed --python 3.15 "tox>=4.45" --with tox-uv + - name: 📥 Download coverage data + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + # one directory per test run: runs on different OSes write the same .coverage. file name + pattern: coverage-* + path: ${{ runner.temp }}/coverage-data + - name: 📊 Combine and report coverage + run: tox run -e coverage -- "$RUNNER_TEMP"/coverage-data/* + - name: 📝 Write the coverage summary + run: .tox/coverage/bin/coverage report --format=markdown >> "$GITHUB_STEP_SUMMARY" + env: + COVERAGE_FILE: .tox/.coverage check: name: 🔎 check ${{ matrix.tox_env }} - ${{ matrix.os }} if: github.event_name != 'schedule' || github.repository_owner == 'pypa' runs-on: ${{ matrix.os }} + permissions: + contents: read strategy: fail-fast: false matrix: os: - ubuntu-24.04 - - windows-2025 + - windows-2025-vs2026 tox_env: - dev - docs + - property + - fuzz - readme + - tasks + - type + - type-3.9 - upgrade - zipapp exclude: - - { os: windows-2025, tox_env: readme } - - { os: windows-2025, tox_env: docs } + - {os: windows-2025-vs2026, tox_env: docs} + - {os: windows-2025-vs2026, tox_env: fuzz} + - {os: windows-2025-vs2026, tox_env: readme} + - {os: windows-2025-vs2026, tox_env: tasks} + - {os: windows-2025-vs2026, tox_env: type} + - {os: windows-2025-vs2026, tox_env: type-3.9} steps: - - name: 🚀 Install uv - uses: astral-sh/setup-uv@v4 - - name: 📦 Install tox - run: uv tool install --python-preference only-managed --python 3.14 tox --with tox-uv - name: 📥 Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 + persist-credentials: false + - name: 🏷️ Fetch upstream tags for versioning + shell: bash + run: | + git fetch --force --tags https://github.com/pypa/virtualenv.git + - name: 🚀 Install uv + uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 + with: + python-version: "3.15" + enable-cache: true + cache-dependency-glob: "pyproject.toml" + cache-suffix: ${{ matrix.tox_env }} + - name: 📦 Install tox + run: uv tool install --python-preference only-managed --python 3.15 "tox>=4.45" --with tox-uv - name: 🏗️ Setup check suite - run: tox run -vv --notest --skip-missing-interpreters false -e ${{ matrix.tox_env }} + run: tox run -vv --notest --skip-missing-interpreters false + env: + TOXENV: ${{ matrix.tox_env }} - name: 🏃 Run check for ${{ matrix.tox_env }} - run: tox run --skip-pkg-install -e ${{ matrix.tox_env }} + run: tox run --skip-pkg-install + env: + TOXENV: ${{ matrix.tox_env }} + UPGRADE_ADVISORY: ${{ matrix.tox_env == 'upgrade' && '1' || '' }} + test-queries: + name: 🔎 test custom queries + if: github.event_name != 'schedule' || github.repository_owner == 'pypa' + runs-on: ubuntu-24.04 + permissions: + contents: read # check out the queries and their test fixtures + security-events: read # codeql-action needs it to reach its API endpoints + steps: + - name: 📥 Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: 🔎 Install CodeQL + id: codeql + uses: github/codeql-action/init@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2 + with: + languages: python + - name: 🏃 Run query tests + run: '"$CODEQL" test run .github/codeql/tests' + env: + CODEQL: ${{ steps.codeql.outputs.codeql-path }} + all-checks-pass: + name: ✅ all checks pass + # the gate shares the jobs' condition, so every job it needs has run and a skip counts as a failure + if: always() && (github.event_name != 'schedule' || github.repository_owner == 'pypa') + needs: [test, coverage, check, test-queries] + runs-on: ubuntu-24.04 + permissions: {} + steps: + - name: ✅ Require every job to succeed + uses: re-actors/alls-green@b5b5b37504aa4183270bd3d855c52a67f212be35 # v1.3.0 + with: + jobs: ${{ toJSON(needs) }} diff --git a/.github/workflows/codeql.yaml b/.github/workflows/codeql.yaml new file mode 100644 index 000000000..fb8c2f5d9 --- /dev/null +++ b/.github/workflows/codeql.yaml @@ -0,0 +1,39 @@ +name: 🔎 CodeQL +on: + workflow_dispatch: + push: + branches: ["main"] + pull_request: + schedule: + - cron: "0 9 * * 2" +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true +permissions: {} +jobs: + analyze: + name: 🔎 analyze ${{ matrix.language }} + runs-on: ubuntu-24.04 + permissions: + contents: read # check out the code to analyze + security-events: write # upload the analysis results to code scanning + strategy: + fail-fast: false + matrix: + language: [python, actions] + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Initialize CodeQL + uses: github/codeql-action/init@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2 + with: + languages: ${{ matrix.language }} + config-file: ./.github/codeql/codeql-config.yml + # the custom pack holds Python queries only, so the actions analysis must not load it + queries: ${{ matrix.language == 'python' && '+./.github/codeql/queries' || '' }} + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2 + with: + category: "/language:${{ matrix.language }}" diff --git a/.github/workflows/dependency-review.yaml b/.github/workflows/dependency-review.yaml new file mode 100644 index 000000000..e8f7a9d43 --- /dev/null +++ b/.github/workflows/dependency-review.yaml @@ -0,0 +1,23 @@ +name: 🔍 dependency review +on: + pull_request: +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true +permissions: {} +jobs: + review: + name: 🔍 dependency review + runs-on: ubuntu-24.04 + permissions: + contents: read # read the dependency graph diff between the base and head commits + steps: + - uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294 # v5.0.0 + with: + # an advisory of any severity in any manifest blocks the change: runtime dependencies run on every install, + # and the release tooling lock runs in jobs that hold publish and attestation permissions + fail-on-severity: low + fail-on-scopes: runtime, development, unknown + # the dependency graph files requirements lock files under runtime scope whatever they pin, so the license + # policy runs on the built wheel and zipapp SBOMs instead; see tasks/license_policy.py + license-check: false diff --git a/.github/workflows/pre-release.yaml b/.github/workflows/pre-release.yaml new file mode 100644 index 000000000..3a374f1c5 --- /dev/null +++ b/.github/workflows/pre-release.yaml @@ -0,0 +1,121 @@ +name: Pre-release +on: + workflow_dispatch: + inputs: + bump: + description: "Version bump type" + required: true + type: choice + options: + - auto + - major + - minor + - patch + default: auto +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false +permissions: {} +jobs: + changelog: + name: Build changelog + # the release commit fast-forwards main from the dispatched commit, so a branch ahead of main would land on main + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-24.04 + permissions: + contents: read # check out the source and let setup-uv download uv with the job token + outputs: + version: ${{ steps.release.outputs.version }} + steps: + - name: Block outbound traffic outside the allowlist + uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + with: + egress-policy: block + disable-sudo-and-containers: true + allowed-endpoints: >- + files.pythonhosted.org:443 github.com:443 pypi.org:443 raw.githubusercontent.com:443 release-assets.githubusercontent.com:443 releases.astral.sh:443 + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + - name: Install the locked version of uv + uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 + with: + python-version: "3.15" + version-file: tasks/release-requirements.txt + enable-cache: false + github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Install the locked release tooling + run: | + uv venv --python 3.15 --python-preference only-managed "$RUNNER_TEMP/release" + uv pip sync --python "$RUNNER_TEMP/release" --require-hashes tasks/release-requirements.txt + - name: Generate changelog, commit, and tag + id: release + env: + BUMP: ${{ inputs.bump }} + run: | + "$RUNNER_TEMP/release/bin/python" tasks/release.py --version "$BUMP" --no-push + echo "version=$(git describe --tags --exact-match)" >> "$GITHUB_OUTPUT" + - name: Store the changelog + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: release-changelog + path: docs/changelog.rst + if-no-files-found: error + retention-days: 1 + push: + name: Push release commit and tag + needs: changelog + runs-on: ubuntu-24.04 + environment: release + permissions: {} # the release app token makes every write + steps: + - name: Block outbound traffic outside the allowlist + uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + with: + egress-policy: block + disable-sudo-and-containers: true + allowed-endpoints: >- + api.github.com:443 + - name: Download the changelog + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: release-changelog + path: ${{ runner.temp }}/changelog + - name: Mint release app token + id: app-token + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + client-id: ${{ secrets.RELEASE_APP_CLIENT_ID }} + private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }} + owner: pypa + repositories: virtualenv + permission-contents: write + # GitHub signs commits the app creates through the API; the commit takes only the changelog from the build job and + # deletes the fragments of the dispatched commit, so nothing else the build job ran can reach main + - name: Commit, tag, and push through the API + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + VERSION: ${{ needs.changelog.outputs.version }} + CHANGELOG: ${{ runner.temp }}/changelog/changelog.rst + run: | + if [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "::error::the build job reported an invalid version: $VERSION" + exit 1 + fi + repo=repos/pypa/virtualenv + base_tree=$(gh api "$repo/git/commits/$GITHUB_SHA" --jq .tree.sha) + blob=$(base64 --wrap=0 "$CHANGELOG" | jq --raw-input --slurp '{content: ., encoding: "base64"}' \ + | gh api "$repo/git/blobs" --input - --jq .sha) + fragments=$(gh api "$repo/contents/docs/changelog?ref=$GITHUB_SHA" --jq '[.[] + | select(.name | test("^[0-9]+[.](feature|bugfix|doc|deprecation|removal)[.]rst$")) + | {path, mode: "100644", type: "blob", sha: null}]') + tree=$(jq --null-input --arg base_tree "$base_tree" --arg blob "$blob" --argjson fragments "$fragments" '{ + base_tree: $base_tree, + tree: ([{path: "docs/changelog.rst", mode: "100644", type: "blob", sha: $blob}] + $fragments) + }' | gh api "$repo/git/trees" --input - --jq .sha) + commit=$(jq --null-input --arg tree "$tree" --arg head "$GITHUB_SHA" --arg message "release $VERSION" \ + '{message: $message, tree: $tree, parents: [$head]}' | gh api "$repo/git/commits" --input - --jq .sha) + gh api --method PATCH "$repo/git/refs/heads/main" -f sha="$commit" -F force=false + gh api --method POST "$repo/git/refs" -f ref="refs/tags/$VERSION" -f sha="$commit" diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index def30f5d9..19af92a03 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,49 +1,336 @@ -name: Release to PyPI +name: Release on: push: - tags: ["*"] - + tags: ["*.*.*"] env: dists-artifact-name: python-package-distributions - +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false +permissions: {} jobs: build: + name: Build runs-on: ubuntu-24.04 + permissions: + contents: read # check out the tagged source + id-token: write # sign attestations with the workflow identity + attestations: write # store the SBOM and provenance attestations + artifact-metadata: write # record the attested artifacts steps: - - name: 📥 Checkout code - uses: actions/checkout@v4 + - name: Block outbound traffic outside the allowlist + uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + with: + egress-policy: block + disable-sudo-and-containers: true + allowed-endpoints: >- + api.github.com:443 files.pythonhosted.org:443 fulcio.sigstore.dev:443 github.com:443 pypi.org:443 raw.githubusercontent.com:443 rekor.sigstore.dev:443 release-assets.githubusercontent.com:443 releases.astral.sh:443 + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 - - name: 🚀 Install the latest version of uv - uses: astral-sh/setup-uv@v4 + persist-credentials: false + - name: Install the locked version of uv + uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 with: - enable-cache: true - cache-dependency-glob: "pyproject.toml" + python-version: "3.15" + version-file: tasks/release-requirements.txt + enable-cache: false github-token: ${{ secrets.GITHUB_TOKEN }} - - name: 📦 Build package - run: uv build --python 3.14 --python-preference only-managed --sdist --wheel . --out-dir dist - - name: 📦 Store the distribution packages - uses: actions/upload-artifact@v4 + - name: Install the locked release tooling + run: | + uv venv --python 3.15 --python-preference only-managed "$RUNNER_TEMP/tools" + uv pip sync --python "$RUNNER_TEMP/tools" --require-hashes tasks/release-requirements.txt + - name: Pin build timestamps to the release commit for reproducible artifacts + run: echo "SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)" >> "$GITHUB_ENV" + - name: Build sdist and wheel + run: | + uv build --python 3.15 --python-preference only-managed --sdist --wheel . --out-dir dist \ + --build-constraints tasks/release-requirements.txt --require-hashes + - name: Extract the SBOM the wheel build embedded + run: uv run --no-project --python 3.15 python -m zipfile -e dist/*.whl sbom-extract/ && cp sbom-extract/*.dist-info/sboms/virtualenv.cdx.json . + - name: Attest the SBOM against the sdist and wheel + uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2 with: - name: ${{ env.dists-artifact-name }} - path: dist/* - - release: - needs: - - build + subject-path: dist/* + sbom-path: virtualenv.cdx.json + - name: Render the SBOM as SPDX + run: '"$RUNNER_TEMP/tools/bin/tox" r -e spdx' + - name: Attest the SPDX SBOM against the sdist and wheel + uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2 + with: + subject-path: dist/* + sbom-path: virtualenv.spdx.json + - name: Build zipapp + run: '"$RUNNER_TEMP/tools/bin/tox" r -e zipapp' + - name: Attest provenance for the zipapp + id: attest + uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 + with: + subject-path: virtualenv.pyz + - name: Rename provenance bundle to the release asset name + env: + BUNDLE_PATH: ${{ steps.attest.outputs.bundle-path }} + run: cp "$BUNDLE_PATH" virtualenv.pyz.intoto.jsonl + - name: Attest the zipapp SBOM against the zipapp + uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2 + with: + subject-path: virtualenv.pyz + sbom-path: virtualenv.pyz.cdx.json + - parallel: + - name: Store the distribution packages + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ${{ env.dists-artifact-name }} + path: dist/* + - name: Store the SBOM + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: virtualenv-sbom + path: | + virtualenv.cdx.json + virtualenv.spdx.json + - name: Store the zipapp + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: virtualenv-zipapp + path: | + virtualenv.pyz + virtualenv.pyz.intoto.jsonl + virtualenv.pyz.cdx.json + publish: + name: Publish + needs: build runs-on: ubuntu-24.04 environment: name: release url: https://pypi.org/project/virtualenv/${{ github.ref_name }} permissions: - id-token: write + contents: write # create the GitHub release + id-token: write # authenticate to PyPI with trusted publishing steps: - - name: 📥 Download all the dists - uses: actions/download-artifact@v4 + - name: Block outbound traffic outside the allowlist + uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + with: + egress-policy: block + allowed-endpoints: >- + api.github.com:443 fulcio.sigstore.dev:443 ghcr.io:443 github.com:443 pkg-containers.githubusercontent.com:443 rekor.sigstore.dev:443 timestamp.sigstore.dev:443 tuf-repo-cdn.sigstore.dev:443 upload.pypi.org:443 uploads.github.com:443 + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + - name: Download all the dists + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: ${{ env.dists-artifact-name }} path: dist/ - - name: 🚀 Publish to PyPI - uses: pypa/gh-action-pypi-publish@v1.13.0 + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 with: attestations: true + - parallel: + - name: Download the zipapp + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: virtualenv-zipapp + - name: Download the SBOMs + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: virtualenv-sbom + # Immutable releases require all assets at creation. + - name: Create GitHub Release + run: | + gh release create "$GITHUB_REF_NAME" virtualenv.pyz virtualenv.pyz.intoto.jsonl virtualenv.pyz.cdx.json \ + virtualenv.cdx.json virtualenv.spdx.json --generate-notes + env: + GH_TOKEN: ${{ github.token }} + - name: Mint get-virtualenv app token + id: app-token + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + client-id: ${{ secrets.RELEASE_APP_CLIENT_ID }} + private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }} + owner: pypa + repositories: get-virtualenv + permission-contents: write + # get-virtualenv main requires signed commits, and GitHub signs commits the app creates through the API + - name: Update get-virtualenv + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + run: | + repo=repos/pypa/get-virtualenv + head=$(gh api "$repo/git/ref/heads/main" --jq .object.sha) + base_tree=$(gh api "$repo/git/commits/$head" --jq .tree.sha) + blob=$(base64 --wrap=0 virtualenv.pyz | jq --raw-input --slurp '{content: ., encoding: "base64"}' \ + | gh api "$repo/git/blobs" --input - --jq .sha) + tree=$(jq --null-input --arg base_tree "$base_tree" --arg blob "$blob" --arg version "$GITHUB_REF_NAME" '{ + base_tree: $base_tree, + tree: [ + {path: "public/virtualenv.pyz", mode: "100644", type: "blob", sha: $blob}, + {path: "public/version.txt", mode: "100644", type: "blob", content: $version} + ] + }' | gh api "$repo/git/trees" --input - --jq .sha) + commit=$(jq --null-input --arg tree "$tree" --arg head "$head" \ + --arg message "update virtualenv to $GITHUB_REF_NAME" '{message: $message, tree: $tree, parents: [$head]}' \ + | gh api "$repo/git/commits" --input - --jq .sha) + gh api --method PATCH "$repo/git/refs/heads/main" -f sha="$commit" -F force=false + verify: + name: Verify + needs: publish + runs-on: ubuntu-24.04 + permissions: + contents: read # check out the tooling lock and download the published zipapp from the release + attestations: read # verify the zipapp provenance + steps: + - name: Block outbound traffic outside the allowlist + uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + with: + egress-policy: block + disable-sudo-and-containers: true + allowed-endpoints: >- + api.github.com:443 files.pythonhosted.org:443 github.com:443 pypi.org:443 raw.githubusercontent.com:443 release-assets.githubusercontent.com:443 releases.astral.sh:443 tmaproduction.blob.core.windows.net:443 tuf-repo-cdn.sigstore.dev:443 tuf-repo.github.com:443 + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Install the locked version of uv + uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 + with: + python-version: "3.15" + version-file: tasks/release-requirements.txt + enable-cache: false + - name: Install the locked verification tooling + run: | + uv venv --python 3.15 --python-preference only-managed "$RUNNER_TEMP/tools" + uv pip sync --python "$RUNNER_TEMP/tools" --require-hashes tasks/release-requirements.txt + - name: Download expected distributions + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: ${{ env.dists-artifact-name }} + path: expected/ + - name: Download expected SBOMs + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: virtualenv-sbom + path: expected/ + - name: Prepare verification directory + run: mkdir published + - name: Download and verify published zipapp + id: zipapp + background: true + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release download "$GITHUB_REF_NAME" --repo pypa/virtualenv --pattern virtualenv.pyz \ + --pattern virtualenv.pyz.intoto.jsonl --pattern virtualenv.pyz.cdx.json --dir published + gh attestation verify published/virtualenv.pyz --repo pypa/virtualenv \ + --bundle published/virtualenv.pyz.intoto.jsonl \ + --signer-workflow pypa/virtualenv/.github/workflows/release.yaml \ + --source-ref "$GITHUB_REF" --source-digest "$GITHUB_SHA" --deny-self-hosted-runners + unzip -p published/virtualenv.pyz virtualenv.pyz.cdx.json | cmp - published/virtualenv.pyz.cdx.json + gh attestation verify published/virtualenv.pyz --repo pypa/virtualenv \ + --predicate-type https://cyclonedx.org/bom \ + --signer-workflow pypa/virtualenv/.github/workflows/release.yaml \ + --source-ref "$GITHUB_REF" --source-digest "$GITHUB_SHA" --deny-self-hosted-runners --format json | + jq --exit-status --slurpfile asset published/virtualenv.pyz.cdx.json \ + 'any(.[]; .verificationResult.statement.predicate == $asset[0])' + - name: Download and verify published distributions + run: | + curl --fail --silent --show-error --retry 5 --retry-all-errors \ + "https://pypi.org/pypi/virtualenv/$GITHUB_REF_NAME/json" --output release.json + for expected in expected/*.whl expected/*.tar.gz; do + filename="$(basename "$expected")" + url="$(jq --exit-status --raw-output --arg filename "$filename" \ + '.urls[] | select(.filename == $filename) | .url' release.json)" + curl --fail --silent --show-error --retry 5 --retry-all-errors --output "published/$filename" -- "$url" + cmp "$expected" "published/$filename" + "$RUNNER_TEMP/tools/bin/pypi-attestations" verify pypi --repository https://github.com/pypa/virtualenv \ + "published/$filename" + done + - name: Download and verify published SBOMs + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release download "$GITHUB_REF_NAME" --repo pypa/virtualenv \ + --pattern virtualenv.cdx.json --pattern virtualenv.spdx.json --dir published + cmp expected/virtualenv.spdx.json published/virtualenv.spdx.json + unzip -p published/*.whl '*.dist-info/sboms/virtualenv.cdx.json' | cmp - published/virtualenv.cdx.json + "$RUNNER_TEMP/tools/bin/pyspdxtools" --infile published/virtualenv.spdx.json + for artifact in published/*.whl published/*.tar.gz; do + for sbom in cdx:https://cyclonedx.org/bom spdx:https://spdx.dev/Document/v2.3; do + gh attestation verify "$artifact" --repo pypa/virtualenv --predicate-type "${sbom#*:}" \ + --signer-workflow pypa/virtualenv/.github/workflows/release.yaml \ + --source-ref "$GITHUB_REF" --source-digest "$GITHUB_SHA" --deny-self-hosted-runners --format json | + jq --exit-status --slurpfile asset "published/virtualenv.${sbom%%:*}.json" \ + 'any(.[]; .verificationResult.statement.predicate == $asset[0])' + done + done + - wait: zipapp + - name: Create environments from verified distributions + run: | + for artifact in published/*.whl published/*.tar.gz; do + installation="$RUNNER_TEMP/$(basename "$artifact")" + uv venv --python 3.15 "$installation" + uv pip install --no-cache --python "$installation/bin/python" "$artifact" + "$installation/bin/python" -I -m virtualenv --no-download --no-periodic-update "$installation/created" + "$installation/created/bin/python" -I -m pip --version + done + uv run --no-project --python 3.15 python -I published/virtualenv.pyz \ + --no-download --no-periodic-update "$RUNNER_TEMP/zipapp-created" + "$RUNNER_TEMP/zipapp-created/bin/python" -I -m pip --version + verify-bootstrap: + name: Verify bootstrap.pypa.io + needs: verify + runs-on: ubuntu-24.04 + timeout-minutes: 150 + permissions: + contents: read # download the published zipapp from the release + attestations: read # verify the served zipapp provenance + steps: + - name: Download the verified zipapp + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release download "$GITHUB_REF_NAME" --repo pypa/virtualenv \ + --pattern virtualenv.pyz --pattern virtualenv.pyz.intoto.jsonl --dir published + - name: Download and verify the zipapp served from bootstrap.pypa.io + env: + GH_TOKEN: ${{ github.token }} + run: | + # pypa/bootstrap polls get-virtualenv every 15 minutes, then rebuilds its image and purges the CDN; the + # per-Python paths that track the latest release are symlinks, the rest pin zipapps for retired Pythons + paths="virtualenv.pyz virtualenv/virtualenv.pyz $(gh api 'repos/pypa/get-virtualenv/git/trees/main?recursive=1' \ + --jq '.tree[] | select(.mode == "120000" and (.path | test("^public/[0-9.]+/virtualenv[.]pyz$"))) + | .path | sub("^public/"; "virtualenv/")')" + deadline=$((SECONDS + 7200)) + for path in $paths; do + mkdir -p "bootstrap/$(dirname "$path")" + until curl --fail --silent --show-error --max-time 300 --retry 3 --retry-all-errors \ + --output "bootstrap/$path" "https://bootstrap.pypa.io/$path" && cmp --silent published/virtualenv.pyz "bootstrap/$path"; do + if ((SECONDS >= deadline)); then + served="$(curl --silent --max-time 30 https://bootstrap.pypa.io/virtualenv/version.txt || true)" + echo "::error::https://bootstrap.pypa.io/$path does not serve $GITHUB_REF_NAME after 2 hours (version.txt: ${served:-unavailable})" + exit 1 + fi + sleep 60 + done + gh attestation verify "bootstrap/$path" --repo pypa/virtualenv \ + --bundle published/virtualenv.pyz.intoto.jsonl \ + --signer-workflow pypa/virtualenv/.github/workflows/release.yaml \ + --source-ref "$GITHUB_REF" --source-digest "$GITHUB_SHA" --deny-self-hosted-runners + done + recovery: + name: Recovery + if: ${{ always() && needs.build.result == 'success' && needs.publish.result == 'failure' }} + needs: + - build + - publish + runs-on: ubuntu-24.04 + permissions: {} + steps: + - name: Report partial publication + run: | + echo "::error::Publication failed. Inspect PyPI, the GitHub release, and get-virtualenv before retrying." + echo "Publication may have succeeded on some destinations. Keep the release tag and commit." >> "$GITHUB_STEP_SUMMARY" + echo "Follow the partial-publication recovery instructions in docs/development.rst." >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/scorecard.yaml b/.github/workflows/scorecard.yaml new file mode 100644 index 000000000..ab40a6206 --- /dev/null +++ b/.github/workflows/scorecard.yaml @@ -0,0 +1,38 @@ +name: 🔎 scorecard +on: + branch_protection_rule: + schedule: + - cron: "30 7 * * 2" + push: + branches: ["main"] + workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true +permissions: {} +jobs: + analysis: + name: 🔎 scorecard analysis + if: github.repository_owner == 'pypa' + runs-on: ubuntu-26.04 + permissions: + actions: read # read workflow runs for the Packaging check + checks: read # read check runs for the CI-Tests and SAST checks + contents: read # check out and scan the repository + id-token: write # authenticate the results published to the scorecard API + issues: read # read issue activity for the Maintained check + pull-requests: read # read reviews for the Code-Review check + security-events: write # upload the SARIF results to code scanning + statuses: read # read commit statuses for the CI-Tests check + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: ossf/scorecard-action@2d1146689b8cda280b9bc96326124645441f03bc # v2.4.4 + with: + results_file: results.sarif + results_format: sarif + publish_results: true + - uses: github/codeql-action/upload-sarif@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2 + with: + sarif_file: results.sarif diff --git a/.github/workflows/upgrade.yaml b/.github/workflows/upgrade.yaml new file mode 100644 index 000000000..299fb26f0 --- /dev/null +++ b/.github/workflows/upgrade.yaml @@ -0,0 +1,170 @@ +name: Upgrade embedded dependencies +on: + schedule: + - cron: "0 10 * * *" # daily at 10:00 UTC + - cron: "0 11 * * 1" # Mondays at 11:00 UTC, after the daily run, to also bump the CI test tools + workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false +permissions: + contents: read +jobs: + generate: + name: Upgrade embedded pip/setuptools/wheel + if: github.repository_owner == 'pypa' + runs-on: ubuntu-24.04 + outputs: + publish: ${{ steps.upgrade.outputs.changed == 'true' && steps.age-check.outputs.skip == 'false' }} + steps: + - name: Block outbound traffic outside the allowlist + uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + with: + egress-policy: block + disable-sudo-and-containers: true + allowed-endpoints: >- + api.github.com:443 files.pythonhosted.org:443 github.com:443 pypi.org:443 raw.githubusercontent.com:443 registry.npmjs.org:443 release-assets.githubusercontent.com:443 releases.astral.sh:443 + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + - name: Install uv + uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 + with: + python-version: "3.15" + enable-cache: false # the egress allowlist above blocks the cache service + - name: Install tox + run: uv tool install --python-preference only-managed --python 3.15 "tox>=4.32" --with tox-uv + - name: Fetch upstream tags for versioning + run: git fetch --force --tags https://github.com/pypa/virtualenv.git + - name: Pin the newest CI test tools + id: ci-tools + if: github.event_name == 'workflow_dispatch' || github.event.schedule == '0 11 * * 1' + env: + GH_TOKEN: ${{ github.token }} + run: uv run --no-project --python 3.15 python tasks/upgrade_ci_tools.py + - name: Run upgrade + id: upgrade + env: + UPGRADE_ADVISORY: "1" + CI_TOOLS_OUTCOME: ${{ steps.ci-tools.outcome }} + run: | + tox run -e upgrade + # hold back releases younger than the seven days check_wheel_age.py demands of embedded wheels + UV_EXCLUDE_NEWER="7 days" uv pip compile pyproject.toml --universal --python-version 3.9 --upgrade -o pylock.zipapp.toml + if ! UV_EXCLUDE_NEWER="7 days" uv pip compile --group pyproject.toml:release --universal --python-version 3.15 \ + --generate-hashes --upgrade -o tasks/release-requirements.txt; then + uv pip compile --group pyproject.toml:release --universal --python-version 3.15 \ + --generate-hashes --upgrade --no-build -o "$RUNNER_TEMP/release-requirements.txt" + echo "changed=false" >> "$GITHUB_OUTPUT" + echo "### Waiting for release dependencies to satisfy the seven-day age filter" >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + git add -- src/virtualenv/seed/wheels/embed THIRD-PARTY-NOTICES.md docs/changelog pylock.zipapp.toml \ + tasks/release-requirements.txt + if [ "$CI_TOOLS_OUTCOME" = "success" ]; then + git add -- tasks/ci-tools.json + fi + if ! git diff --cached --quiet; then + echo "changed=true" >> "$GITHUB_OUTPUT" + echo "### Embedded dependency changes" >> "$GITHUB_STEP_SUMMARY" + git diff --cached --stat >> "$GITHUB_STEP_SUMMARY" + else + echo "changed=false" >> "$GITHUB_OUTPUT" + echo "### All embedded dependencies are up to date" >> "$GITHUB_STEP_SUMMARY" + fi + - name: Check PyPI release age + if: steps.upgrade.outputs.changed == 'true' + id: age-check + run: | + uv run --no-project --python 3.15 python tasks/check_wheel_age.py >> "$GITHUB_OUTPUT" + - name: Prepare upgrade patch + if: steps.upgrade.outputs.changed == 'true' && steps.age-check.outputs.skip == 'false' + run: git diff --cached --binary --no-ext-diff --no-textconv > "$RUNNER_TEMP/upgrade.patch" + - name: Store upgrade patch + if: steps.upgrade.outputs.changed == 'true' && steps.age-check.outputs.skip == 'false' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: upgrade-patch + path: ${{ runner.temp }}/upgrade.patch + if-no-files-found: error + retention-days: 1 + publish: + name: Open upgrade pull request + needs: generate + if: needs.generate.outputs.publish == 'true' + runs-on: ubuntu-24.04 + environment: upgrade + permissions: + actions: write # dispatch CI on the upgrade branch + contents: write # let create-pull-request manage the upgrade branch + pull-requests: write # open the upgrade pull request + steps: + - name: Block outbound traffic outside the allowlist + uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + with: + egress-policy: block + disable-sudo-and-containers: true + allowed-endpoints: >- + api.github.com:443 github.com:443 release-assets.githubusercontent.com:443 releases.astral.sh:443 + - name: Checkout trusted generator revision + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + fetch-depth: 0 + persist-credentials: true # zizmor: ignore[artipacked] create-pull-request and the changelog rename push with it + - name: Download upgrade patch + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: upgrade-patch + path: ${{ runner.temp }}/upgrade-patch + - name: Install uv + uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 + with: + python-version: "3.15" + enable-cache: false + - name: Validate and apply upgrade patch + run: uv run --no-project --python 3.15 python tasks/apply_upgrade_patch.py "$RUNNER_TEMP/upgrade-patch/upgrade.patch" + - name: Create Pull Request + id: cpr + uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 + with: + commit-message: "Upgrade embedded dependencies" + # the checkout pins github.sha, a detached HEAD, so the action cannot infer the base branch + base: main + branch: auto/upgrade-embedded-deps + delete-branch: true + title: "Upgrade embedded pip/setuptools/wheel and CI test tools" + body: | + Automated upgrade of embedded pip, setuptools, and wheel dependencies, of the locked release tooling, of the GraalPy, RustPython and Nushell releases CI tests against, and of the Mermaid version the documentation loads. + + This PR was created automatically by the [upgrade workflow](https://github.com/${{ github.repository }}/actions/workflows/upgrade.yaml). + labels: | + dependencies + - name: Rename changelog with PR number + if: steps.cpr.outputs.pull-request-number + run: | + pr_number="${STEPS_CPR_OUTPUTS_PULL_REQUEST_NUMBER}" + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git fetch origin auto/upgrade-embedded-deps + git checkout -B auto/upgrade-embedded-deps FETCH_HEAD + if [ ! -f docs/changelog/u.bugfix.rst ]; then + exit 0 + fi + mv docs/changelog/u.bugfix.rst "docs/changelog/${pr_number}.bugfix.rst" + git add docs/changelog/ + git commit -m "Rename changelog to ${pr_number}.bugfix.rst" + git push origin HEAD:auto/upgrade-embedded-deps + env: + STEPS_CPR_OUTPUTS_PULL_REQUEST_NUMBER: ${{ steps.cpr.outputs.pull-request-number }} + - name: Run CI on the upgrade branch + if: steps.cpr.outputs.pull-request-number + # events from GITHUB_TOKEN start no workflows, except workflow_dispatch, so the pull request gets no CI otherwise + run: | + gh workflow run check.yaml --ref auto/upgrade-embedded-deps + gh workflow run codeql.yaml --ref auto/upgrade-embedded-deps + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} diff --git a/.gitignore b/.gitignore index b799ed285..caa464e53 100644 --- a/.gitignore +++ b/.gitignore @@ -27,4 +27,5 @@ dist *wheel-store* Dockerfile* +!.oss-scanner/Dockerfile .dockerignore diff --git a/.markdownlint.yaml b/.markdownlint.yaml index 33a1615f7..b0571c38a 100644 --- a/.markdownlint.yaml +++ b/.markdownlint.yaml @@ -3,7 +3,6 @@ MD013: headers: false line_length: 120 tables: false - MD046: style: fenced no-emphasis-as-header: false diff --git a/.oss-scanner/Dockerfile b/.oss-scanner/Dockerfile new file mode 100644 index 000000000..215687a34 --- /dev/null +++ b/.oss-scanner/Dockerfile @@ -0,0 +1,37 @@ +# virtualenv installed from source the way CI installs it (.github/workflows/check.yaml and tox.toml in the virtualenv +# repository), with the shells whose activation scripts the tests run. +FROM python:3.14-bookworm +ENV DEBIAN_FRONTEND=noninteractive \ + PIP_DISABLE_PIP_VERSION_CHECK=1 +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates curl git gnupg bash csh tcsh fish \ + && rm -rf /var/lib/apt/lists/* + +# Nushell from the repository CI uses, and PowerShell from Microsoft's, because Debian packages neither. +RUN install -m 0755 -d /etc/apt/keyrings \ + && curl -fsSL https://apt.fury.io/nushell/gpg.key | gpg --dearmor -o /etc/apt/keyrings/fury-nushell.gpg \ + && echo "deb [signed-by=/etc/apt/keyrings/fury-nushell.gpg] https://apt.fury.io/nushell/ /" \ + > /etc/apt/sources.list.d/fury.list \ + && curl -fsSL -o /tmp/packages-microsoft-prod.deb \ + https://packages.microsoft.com/config/debian/12/packages-microsoft-prod.deb \ + && dpkg -i /tmp/packages-microsoft-prod.deb && rm /tmp/packages-microsoft-prod.deb \ + && apt-get update && apt-get install -y --no-install-recommends nushell powershell \ + && rm -rf /var/lib/apt/lists/* + +# A virtual environment on PATH, as tox gives the tests: some tests skip or change behavior under a system Python. +RUN python -m venv /opt/venv +ENV PATH=/opt/venv/bin:$PATH + +# scanner contract: the checkout lives inside the image, at /src +COPY . /src +WORKDIR /src + +# The package with the test, property (Hypothesis) and fuzz (Atheris) dependency groups, plus xonsh for its activator +# tests. hatch-vcs reads the version from git, so pin one when the checkout has no history. +RUN pip install --upgrade pip \ + && if ! git -C /src rev-parse --git-dir >/dev/null 2>&1; then export SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0; fi \ + && pip install -e . --group test --group property --group fuzz xonsh + +# Run the fast unit tests so you know the image works, but do not let a failing test abort the build. +RUN python -m pytest tests/unit --skip-slow -n auto -p no:randomly \ + || echo "WARNING: virtualenv unit tests reported failures" diff --git a/.oss-scanner/threat_model.md b/.oss-scanner/threat_model.md new file mode 100644 index 000000000..c99ca0190 --- /dev/null +++ b/.oss-scanner/threat_model.md @@ -0,0 +1,105 @@ +# Threat model + +The project's own policy is [.github/SECURITY.md](../.github/SECURITY.md), and the full threat model with its assurance +case is [.github/THREAT_MODEL.md](../.github/THREAT_MODEL.md). This file summarizes them for the scanner. Where they +disagree, SECURITY.md wins. + +## What this project does and where untrusted input enters + +virtualenv creates Python virtual environments. It runs as the `virtualenv` command, as `python -m virtualenv`, as a +zipapp, and through the Python API (`virtualenv.cli_run`), which tox and hatch call with values they read from project +configuration. A run has four stages, each a plugin loaded from entry points (`src/virtualenv/run/plugin/`): + +1. Discovery (`src/virtualenv/discovery/`) finds the target interpreter, runs it in a subprocess + (`discovery/py_info.py`, `discovery/cached_py_info.py`) and caches the result in the app-data directory. +1. Creation (`src/virtualenv/create/`) writes the environment: interpreter links or copies, `pyvenv.cfg` + (`create/pyenv_cfg.py`), `_virtualenv.py`/`.pth` into site-packages, `CACHEDIR.TAG` and `.gitignore`. +1. Seeding (`src/virtualenv/seed/`) installs pip and setuptools from the wheels embedded in `seed/wheels/embed/`, from + newer wheels the periodic update downloaded into app-data (`seed/wheels/periodic_update.py`), from + `--extra-search-dir`, or from PyPI with `--download` (`seed/wheels/acquire.py`). The app-data seeder extracts wheels + into a cached image and links or copies them (`seed/embed/via_app_data/`). +1. Activation (`src/virtualenv/activation/`) renders one script per shell from templates: bash, batch, csh, fish, + Nushell, PowerShell, xonsh and Python (`activate_this.py`). It substitutes the destination path, the prompt, the + interpreter path and the Tcl/Tk paths. + +virtualenv runs with the rights of the user who starts it. The user and their environment are trusted: command line +options, `VIRTUALENV_*` environment variables (`config/env_var.py`), the `virtualenv.ini` file (`config/ini.py`), `PATH` +and the interpreters found on it, pip's configuration, installed plugins, `--extra-search-dir` contents and the user's +app-data directory. + +Untrusted input is: + +- values a caller passes on someone else's behalf, chiefly the destination path and `--prompt`. tox and hatch derive + these from project metadata the user did not write, so they may contain quotes, `$`, backticks, `%`, `!`, newlines, + and non-ASCII text; +- wheel bytes and metadata from PyPI, fetched by the periodic update and by `--download`, which virtualenv checks + against the SHA-256 digest PyPI's JSON API publishes; +- wheel archives being extracted into app-data (entry names, symlinks); +- the embedded wheels, which virtualenv hashes against the table in `seed/wheels/embed/__init__.py` before first use, + including when reading from inside the zipapp (`util/zipapp.py`). + +## Components that matter most / least + +Most: + +- the activators and their quoting, in `src/virtualenv/activation/*/__init__.py`, the templates beside them, and + `activation/via_template.py`. The batch activator also has logic to refuse paths it cannot quote; +- the `pyvenv.cfg` writer (`create/pyenv_cfg.py`) and the line boundary helper (`util/text.py`); +- wheel acquisition and integrity checks: `seed/wheels/acquire.py`, `seed/wheels/bundle.py`, + `seed/wheels/periodic_update.py`, `seed/wheels/embed/__init__.py`; +- wheel extraction and linking into app-data and the environment: `seed/embed/via_app_data/`, `app_data/`, + `util/zipapp.py`; +- filesystem writes that could escape the destination or follow a symlink: `create/creator.py`, `util/path/`, + `util/lock.py`; +- arguments passed to `pip download` and other subprocesses (`seed/embed/pip_invoke.py`, `util/subprocess/`). + +Least: discovery of interpreters on `PATH` (the interpreter is trusted), `report.py`, the CLI help text, and everything +under `tasks/` (release and CI tooling) and `docs/`. Test code and fixtures are out of scope. + +## How to exercise it + +The repository is at `/src`, installed in editable mode into a virtual environment at `/opt/venv` that is on `PATH`, +with the `test`, `property` and `fuzz` dependency groups. bash, csh, tcsh, fish, nu, pwsh and xonsh are installed, so +the activator tests run the generated scripts in real shells. + +- Create an environment: `virtualenv --no-periodic-update --prompt '' '/tmp//venv'`, then source the + activation script in the matching shell. `--activators bash,fish` limits which scripts are written. +- Unit tests: `python -m pytest tests/unit -n auto`. Activator tests are in `tests/unit/activation/` (one file per + shell), `pyvenv.cfg` in `tests/unit/create/test_creator.py`, seeding and wheel checks in `tests/unit/seed/`. +- Property tests: `python -m pytest tests/property -m property`, covering batch quoting and the `pyvenv.cfg` writer. +- Fuzz targets: `python tasks/fuzz_pyenv_cfg.py -runs=20000`, and the same for `tasks/fuzz_nushell_quote.py` and + `tasks/fuzz_powershell_quote.py`. +- Integration tests need `--int`; the zipapp test may need network: `python -m pytest tests/integration --int`. + +The scan has no network, so `--download`, the periodic update and the PyPI digest check can only be exercised with the +mocks the tests use. + +## How you rate severity + +The levels are those in `.github/INCIDENT_RESPONSE.md`. + +- High: attacker-chosen data becomes code on a normal path with no unusual setup. A destination path or `--prompt` that + makes any activation script run a command when sourced is high (GHSA-x78j-v8h9-3j2q in `activate.bat`, + GHSA-p58f-9548-mpm2 in bash and fish). So is a downloaded seed wheel that is used despite failing or skipping the + SHA-256 check without the user opting out, and wheel extraction that writes outside app-data. +- Medium: real damage that needs a race, a specific configuration or local timing, such as a TOCTOU gap in directory + creation (GHSA-597g-3phw-6986), or a caller value that adds a key or line to `pyvenv.cfg` (GHSA-9h9j-4vrj-gf7g). + Writing outside the destination through a symlink, or creating files with wider permissions than intended, is medium + unless it gives code execution. +- Low: an integrity or correctness failure with narrow effect (GHSA-94p9-xgh2-xp45), and denial of service such as a + crash or a hang on a crafted path or wheel. + +## Anything to leave alone + +These are out of scope in SECURITY.md or accepted risks in THREAT_MODEL.md: + +- an attacker who already runs code as the user, or can write to the user's app-data directory, `PATH`, interpreters, + `virtualenv.ini`, `VIRTUALENV_*` variables or `--extra-search-dir`. Cached wheels and cached interpreter info in + app-data are read back without re-hashing by design; +- shell syntax in the command the user types to launch virtualenv, as opposed to data that reaches a generated file; +- code in pip, setuptools or packages installed into the environment, and plugins registered through entry points; +- skipping the PyPI digest check when `PIP_INDEX_URL`, `PIP_EXTRA_INDEX_URL` or `PIP_INDEX` is set, and the unverified + TLS fallback behind `VIRTUALENV_PERIODIC_UPDATE_INSECURE`; +- a privileged user running virtualenv on a directory a less privileged user can write; +- `pyvenv.cfg` recording the interpreter path and the creating command; +- the release workflows and CI configuration, test code and fixtures, and Python versions virtualenv no longer supports. diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 58acc42ec..1e14f88b5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,3 +1,5 @@ +default_language_version: + python: python3.14 repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v6.0.0 @@ -5,37 +7,65 @@ repos: - id: end-of-file-fixer - id: trailing-whitespace - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.35.0 + rev: "0.38.2" hooks: - id: check-github-workflows args: ["--verbose"] - repo: https://github.com/codespell-project/codespell - rev: v2.4.1 + rev: v2.4.3 hooks: - id: codespell args: ["--write-changes"] - - repo: https://github.com/tox-dev/tox-ini-fmt - rev: "1.7.0" + - repo: https://github.com/tox-dev/tox-toml-fmt + rev: "v1.11.1" hooks: - - id: tox-ini-fmt - args: ["-p", "fix"] + - id: tox-toml-fmt - repo: https://github.com/tox-dev/pyproject-fmt - rev: "v2.11.1" + rev: "v2.30.3" hooks: - id: pyproject-fmt - repo: https://github.com/astral-sh/ruff-pre-commit - rev: "v0.14.6" + rev: "v0.17.0" hooks: - id: ruff-format - id: ruff args: ["--fix", "--unsafe-fixes", "--exit-non-zero-on-fix"] - - repo: https://github.com/rbubley/mirrors-prettier - rev: "v3.6.2" + - repo: https://github.com/google/yamlfmt + rev: "v0.21.0" hooks: - - id: prettier + - id: yamlfmt + - repo: local + hooks: + - id: mdformat + name: mdformat + entry: mdformat --wrap 120 + language: python + types: [markdown] additional_dependencies: - - prettier@3.3.3 - - "@prettier/plugin-xml@3.4.1" + - mdformat-gfm>=1 + - mdformat-ruff>=0.1.3 + # towncrier skips a fragment whose type it does not know, so a typo or a dropped type like + # misc would otherwise vanish from the changelog without any warning + - id: changelog-fragment-name + name: changelog fragment uses an allowed type + entry: "name changelog fragments ..rst, type one of: feature, bugfix, doc, deprecation, removal" + language: fail + files: '^docs/changelog/' + exclude: '^docs/changelog/(examples\.rst|template\.jinja2|\d+\.(feature|bugfix|doc|deprecation|removal)\.rst)$' + - repo: https://github.com/LilSpazJoekp/docstrfmt + rev: v2.2.1 + hooks: + - id: docstrfmt + args: ["-l", "120"] + additional_dependencies: ["sphinx>=9.1"] + - repo: https://github.com/zizmorcore/zizmor-pre-commit + rev: v1.30.1 + hooks: + - id: zizmor + - repo: https://github.com/crate-ci/typos + rev: v1.51.1 + hooks: + - id: typos - repo: meta hooks: - id: check-hooks-apply diff --git a/.readthedocs.yml b/.readthedocs.yml index ab3011302..84f75c677 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -1,15 +1,8 @@ version: 2 build: - os: ubuntu-22.04 - tools: - python: "3" -python: - install: - - method: pip - path: . - extra_requirements: - - docs -sphinx: - builder: html - configuration: docs/conf.py - fail_on_warning: true + os: ubuntu-lts-latest + tools: {} + commands: + - curl -LsSf https://astral.sh/uv/install.sh | sh + - ~/.local/bin/uv tool install tox --with tox-uv -p 3.15 --managed-python + - ~/.local/bin/tox run -e docs -- diff --git a/LICENSE b/LICENSE index be9700d61..99598eb01 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2020-202x The virtualenv developers +Copyright (c) 2020-present The virtualenv developers Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the diff --git a/README.md b/README.md index 87f45e077..3d7626e54 100644 --- a/README.md +++ b/README.md @@ -3,15 +3,17 @@ [![PyPI](https://img.shields.io/pypi/v/virtualenv?style=flat-square)](https://pypi.org/project/virtualenv) [![PyPI - Implementation](https://img.shields.io/pypi/implementation/virtualenv?style=flat-square)](https://pypi.org/project/virtualenv) [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/virtualenv?style=flat-square)](https://pypi.org/project/virtualenv) -[![Documentation](https://readthedocs.org/projects/virtualenv/badge/?version=latest&style=flat-square)](http://virtualenv.pypa.io) +[![Documentation](https://readthedocs.org/projects/virtualenv/badge/?version=latest&style=flat-square)](https://virtualenv.pypa.io) [![Discord](https://img.shields.io/discord/803025117553754132)](https://discord.gg/pypa) [![Downloads](https://static.pepy.tech/badge/virtualenv/month)](https://pepy.tech/project/virtualenv) [![PyPI - License](https://img.shields.io/pypi/l/virtualenv?style=flat-square)](https://opensource.org/licenses/MIT) [![check](https://github.com/pypa/virtualenv/actions/workflows/check.yaml/badge.svg)](https://github.com/pypa/virtualenv/actions/workflows/check.yaml) +[![OpenSSF Scorecard](https://api.scorecard.dev/projects/github.com/pypa/virtualenv/badge)](https://scorecard.dev/viewer/?uri=github.com/pypa/virtualenv) +[![OpenSSF Best Practices](https://www.bestpractices.dev/projects/14689/badge)](https://www.bestpractices.dev/projects/14689) A tool for creating isolated `virtual` python environments. -- [Installation](https://virtualenv.pypa.io/en/latest/installation.html) +- [Installation](https://virtualenv.pypa.io/en/latest/how-to/install.html) - [Documentation](https://virtualenv.pypa.io) - [Changelog](https://virtualenv.pypa.io/en/latest/changelog.html) - [Issues](https://github.com/pypa/virtualenv/issues) diff --git a/THIRD-PARTY-NOTICES.md b/THIRD-PARTY-NOTICES.md new file mode 100644 index 000000000..b50c4f479 --- /dev/null +++ b/THIRD-PARTY-NOTICES.md @@ -0,0 +1,62 @@ + + +# Third-party notices + +virtualenv is distributed under the MIT License, see `LICENSE`. The wheels under `src/virtualenv/seed/wheels/embed/` are +redistributed unmodified and remain under their own licenses, reproduced below. Each wheel also carries its own license +text inside its `.dist-info/licenses/` directory, including the licenses of anything it in turn vendors. + +## pip + +Bundled as: `pip-26.0.1-py3-none-any.whl`, `pip-26.2.1-py3-none-any.whl` + +Upstream: https://pypi.org/project/pip/ + +``` +Copyright (c) 2008-present The pip developers (see AUTHORS.txt file) + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +``` + +## setuptools + +Bundled as: `setuptools-82.0.1-py3-none-any.whl`, `setuptools-84.0.0-py3-none-any.whl` + +Upstream: https://pypi.org/project/setuptools/ + +``` +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to +deal in the Software without restriction, including without limitation the +rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +sell copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. +``` diff --git a/_typos.toml b/_typos.toml new file mode 100644 index 000000000..aa1104512 --- /dev/null +++ b/_typos.toml @@ -0,0 +1,2 @@ +[default.extend-words] +intoto = "intoto" diff --git a/docs/_static/mermaid-normalize.js b/docs/_static/mermaid-normalize.js new file mode 100644 index 000000000..0b49d1868 --- /dev/null +++ b/docs/_static/mermaid-normalize.js @@ -0,0 +1,29 @@ +(function () { + function dedent(text) { + const lines = text.replace(/\s+$/u, "").split("\n"); + const indents = lines + .filter((line) => line.trim()) + .map((line) => line.match(/^\s*/u)[0].length); + + if (!indents.length) { + return text; + } + + const indent = Math.min(...indents); + return lines.map((line) => line.slice(indent)).join("\n"); + } + + function normalizeMermaidBlocks() { + document.querySelectorAll("pre.mermaid").forEach((block) => { + block.textContent = dedent(block.textContent); + }); + } + + if (document.readyState === "loading") { + document.addEventListener("DOMContentLoaded", normalizeMermaidBlocks, { + once: true, + }); + } else { + normalizeMermaidBlocks(); + } +})(); diff --git a/docs/_static/rtd-search.js b/docs/_static/rtd-search.js new file mode 100644 index 000000000..133e1e31c --- /dev/null +++ b/docs/_static/rtd-search.js @@ -0,0 +1,11 @@ +document.addEventListener("DOMContentLoaded", () => { + const searchInput = document.querySelector( + ".sidebar-search input[type='search']", + ); + if (searchInput) { + searchInput.addEventListener("focus", () => { + document.dispatchEvent(new CustomEvent("readthedocs-search-show")); + searchInput.blur(); + }); + } +}); diff --git a/docs/_static/virtualenv.png b/docs/_static/virtualenv.png new file mode 100644 index 000000000..1f729837d Binary files /dev/null and b/docs/_static/virtualenv.png differ diff --git a/docs/_static/virtualenv.svg b/docs/_static/virtualenv.svg new file mode 100644 index 000000000..090396002 --- /dev/null +++ b/docs/_static/virtualenv.svg @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + + + + + virtualenv + + + + + + + + diff --git a/docs/changelog.rst b/docs/changelog.rst index d23e3e4ed..e8ac81e1b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -1,1063 +1,2166 @@ -Release History -=============== +################# + Release History +################# -.. include:: _draft.rst +.. towncrier-draft-entries:: [UNRELEASED DRAFT] .. towncrier release notes start -v20.35.4 (2025-10-28) ---------------------- +*********************** + v21.14.7 (2026-10-12) +*********************** + +Bugfixes - 21.14.7 +================== + +- Escape the environment name in the bash activator for both of bash's ``PS1`` passes, so a name or ``--prompt`` value + with a backslash before ``$(...)`` or backticks no longer runs as a command each time bash draws the prompt - by + :user:`gaborbernat`. (:issue:`3390`) + +*********************** + v21.14.6 (2026-10-08) +*********************** + +Bugfixes - 21.14.6 +================== + +- Fall back to the bundled seed wheel with periodic update on (the default) when the embed update log holds JSON of the + wrong shape, instead of downloading the wheel or, offline, failing to create the environment - by :user:`darrenhuai`. + (:issue:`3383`) + +*********************** + v21.14.5 (2026-10-02) +*********************** + +Bugfixes - 21.14.5 +================== + +- Fall back to the bundled seed wheel and remove the embed update log when the log in the app data folder holds JSON of + the wrong shape, instead of failing to create the environment - by :user:`pasmud`. (:issue:`3376`) + +*********************** + v21.14.4 (2026-10-02) +*********************** + +Bugfixes - 21.14.4 +================== + +- Fix ``activate`` running commands from a virtual environment name or ``--prompt`` that holds ``$(...)``, backticks or + ``${...}`` when zsh has ``PROMPT_SUBST`` set, and show a ``%`` in the name as typed under zsh (`GHSA-5vjq-rrrf-7h2q + `_); reported by :user:`kemrec`. + (:issue:`3373`) + +*********************** + v21.14.3 (2026-10-01) +*********************** + +Bugfixes - 21.14.3 +================== + +- Honor ``~=`` and every clause of a seed wheel's ``Requires-Python`` when picking a wheel for the target Python, and + skip a wheel whose ``Requires-Python`` is not a valid specifier instead of failing - by :user:`pasmud`. + (:issue:`3369`) + +*********************** + v21.14.2 (2026-10-01) +*********************** + +Bugfixes - 21.14.2 +================== + +- Fix ``pyvenv.cfg`` getting an absolute path in ``python-version``, ``include-system-site-packages`` and the other keys + that hold no path when the working directory has an entry named like the value, such as ``3.14`` for ``virtualenv + 3.14`` or ``true`` with ``--system-site-packages`` - by :user:`darrenhuai`. (:issue:`3366`) +- Fix activation scripts running code from a virtual environment path whose parent directory carries a placeholder name + such as ``__VIRTUAL_NAME__`` (`GHSA-8rjx-v5ww-45pp + `_), and ``activate.fish`` running + commands from a path or ``--prompt`` that holds a backslash before a single quote (`GHSA-c947-3pg5-gm8q + `_); both reported by :user:`Kwstubbs` of + GitHub Security Lab. (:issue:`3367`) + +*********************** + v21.14.1 (2026-09-29) +*********************** + +No significant changes. + +*********************** + v21.14.0 (2026-09-29) +*********************** + +Features - 21.14.0 +================== + +- Record the SPDX license id in the wheel and zipapp SBOMs for bundled packages that declare their license by name or + classifier, such as ``distlib`` and ``python-discovery``, so license scanners can match them. (:issue:`3339`) +- Add the Tidelift and thanks.dev funding links to the PyPI project URLs, next to GitHub Sponsors - by + :user:`gaborbernat`. (:issue:`3340`) + +Bugfixes - 21.14.0 +================== + +- Build the release sdist, wheel and zipapp with the build backend and SBOM tools that + ``tasks/release-requirements.txt`` pins by version and SHA-256, so a rebuild of the tag uses the same versions. + (:issue:`3337`) +- Use PEP 440 version ordering when selecting seed wheels, so pinned pre-releases, post-releases and local versions + resolve to the requested wheel. (:issue:`3361`) + +Improved Documentation - 21.14.0 +================================ + +- Apply the licensing policy to what the wheel and zipapp ship, checked against their SBOMs, and name + ``ALLOWED_LICENSES`` in ``tasks/license_policy.py`` as the list of accepted licenses. (:issue:`3339`) +- Document what a maintainer checks when reviewing a pull request and what the ``main`` ruleset requires before a merge + - by :user:`gaborbernat`. (:issue:`3341`) +- Document how the project selects, obtains and tracks its dependencies, and which dependency findings block a merge or + a release - by :user:`gaborbernat`. (:issue:`3342`) +- Document which static analysis findings block a merge or a release, and how maintainers suppress a finding - by + :user:`gaborbernat`. (:issue:`3343`) +- Document which secrets the project stores, who can read them and when maintainers rotate them - by + :user:`gaborbernat`. (:issue:`3344`) +- Update the roadmap: :PEP:`832` support shipped, and :PEP:`838` waits on the PEP - by :user:`gaborbernat`. + (:issue:`3345`) + +*********************** + v21.13.0 (2026-09-26) +*********************** + +Features - 21.13.0 +================== + +- Add ``Changelog`` and ``Funding`` links to the PyPI project metadata, and replace the ``2020-202x`` placeholder in + ``LICENSE`` with ``2020-present`` so the copyright field of the wheel SBOM reads as a real range - by + :user:`gaborbernat`. (:issue:`3334`) + +Bugfixes - 21.13.0 +================== + +- Escape the curly single quotes ``U+2018``-``U+201B`` in the PowerShell activator's ``quote`` so a virtual environment + prompt, name or Tcl/Tk library path containing them can no longer close the string literal and run commands - by + :user:`gaborbernat`. (:issue:`3325`) +- Stop the bash activator from re-expanding the prompt inside ``PS1``, so a virtual environment name or ``--prompt`` + value containing ``$(...)``, backticks or backslashes no longer runs as a command each time bash draws the prompt - by + :user:`gaborbernat`. (:issue:`3326`) + +Improved Documentation - 21.13.0 +================================ + +- Correct the documented changelog fragment types and example to match the enforced towncrier categories. + (:issue:`3323`) +- List the Unlicense and the SPDX BSD variants in the runtime dependency licensing policy, matching the ``filelock`` + releases virtualenv installs on Python 3.9, and describe the dependency review check that enforces the policy. + (:issue:`3329`) +- Show how to check a release with ``gh release verify`` and ``gh release verify-asset``, and how to rebuild the zipapp + byte for byte with the build tool versions its SBOMs list. Drop the claims that the zipapp build does not pin the + distributions it bundles and that wheels differ between build machines - by :user:`gaborbernat`. (:issue:`3330`) + +*********************** + v21.12.1 (2026-09-24) +*********************** + +Bugfixes - 21.12.1 +================== + +- Limit the :PEP:`832` ``.venv`` redirect to folders holding a ``pyproject.toml`` and no ``.venv`` yet, so ``virtualenv + foo`` in a scratch folder, and tools such as tox or nox building environments through virtualenv, no longer claim a + folder's default environment - by :user:`gaborbernat`. + + - ``--venv-redirect`` writes the redirect in any folder and replaces an earlier virtualenv redirect. + - A flag on the command line overrides the environment variable and the config file in either direction. + (:issue:`3316`) + +*********************** + v21.12.0 (2026-09-24) +*********************** + +Features - 21.12.0 +================== + +- Write the `PEP 838 `_ ``python-version`` key into ``pyvenv.cfg``, holding the + target interpreter's feature release. The new :doc:`reference/files` page covers it alongside every other file a + created environment holds - by :user:`konstin`. (:issue:`3193`) +- Point a ``.venv`` redirect file in the parent folder at the created environment, per `PEP 832 + `_, so editors and type checkers can find it - by :user:`gaborbernat`. + + - virtualenv leaves a ``.venv`` folder alone, and a redirect pointing at an environment it did not create. + - Pass ``--no-venv-redirect`` to opt out. + - The feature is provisional while PEP 832 is a draft: a minor or patch release may change it in backward incompatible + ways to follow the PEP. (:issue:`3204`) + +*********************** + v21.11.1 (2026-09-23) +*********************** + +Bugfixes - 21.11.1 +================== + +- Include the pre-commit configuration and the zipapp lock file in the source distribution, so downstream packagers can + run the test suite from it. (:issue:`3314`) + +*********************** + v21.11.0 (2026-09-23) +*********************** + +Features - 21.11.0 +================== + +- Attach the CycloneDX SBOM and an SPDX 2.3 rendering of it (``virtualenv.cdx.json``, ``virtualenv.spdx.json``) to each + GitHub release, and attest the SPDX document against the sdist and wheel. (:issue:`3299`) +- Describe the zipapp in its own CycloneDX SBOM, which lists virtualenv, the embedded pip and setuptools wheels, and + each bundled dependency with the Python versions that load it, down to a SHA-256 per file. The SBOM sits at the root + of ``virtualenv.pyz``, ships as the ``virtualenv.pyz.cdx.json`` release asset, and GitHub attests it against the + zipapp. (:issue:`3310`) + +Bugfixes - 21.11.0 +================== + +- Refuse a seed wheel downloaded from the default index when virtualenv cannot get its sha256 from PyPI after three + retries, instead of seeding it unverified. Seeding falls back to the bundled wheel unless a specific version was + requested. (:issue:`3302`) +- Build the zipapp from the dependency wheels that ``pylock.zipapp.toml`` pins by URL and sha256, so a release bundles + only files matching the committed hashes. (:issue:`3306`) +- Leave the build machine out of the wheel and zipapp SBOMs, so the wheel rebuilds byte for byte on any operating system + and architecture given the same source, ``SOURCE_DATE_EPOCH``, Python patch version and build backend versions. Zipapp + entries now carry the ``SOURCE_DATE_EPOCH`` timestamp and fixed permissions instead of the build time. (:issue:`3311`) + +Improved Documentation - 21.11.0 +================================ + +- Document the backup maintainer and the two-factor authentication and access review expectations for maintainers. + (:issue:`3290`) +- Link the Carl Meyer talk on the home page over HTTPS - by :user:`gaborbernat`. (:issue:`3300`) +- Add a roadmap to the development guide listing what the project plans to do, and not do, through September 2027 - by + :user:`gaborbernat`. (:issue:`3304`) +- Document what a release publishes and how to verify each file, covering the PyPI attestations, the zipapp and its + provenance, the CycloneDX and SPDX SBOMs of the wheel, the zipapp SBOM, and rebuilding the sdist from its tag. + (:issue:`3305`) + +*********************** + v21.10.0 (2026-09-22) +*********************** + +Features - 21.10.0 +================== + +- Include versioned identities for packages vendored inside seed wheels in the embedded SBOM. (:issue:`3281`) +- Allow ``filelock`` 4.x on Python 3.10+ by relaxing the upper bound from ``<4`` to ``<5`` - by :user:`r3wretrhy`. + (:issue:`3285`) + +Bugfixes - 21.10.0 +================== + +- Skip Windows batch activation scripts when environment or Tcl/Tk paths contain ``^`` or ``!``, since quoting changes + those paths. Other activation scripts remain available. (:issue:`3280`) + +********************** + v21.9.1 (2026-09-22) +********************** + +Bugfixes - 21.9.1 +================= + +- Correct SBOM CSV parsing and unresolved dependency relationships, and omit CI run identifiers from reproducible + builds. (:issue:`3278`) + +********************** + v21.9.0 (2026-09-20) +********************** + +Features - 21.9.0 +================= + +- The embedded SBOM now describes the root component's license, copyright, maintainers and project links, each bundled + wheel from its own metadata, the declared runtime dependencies, and the full build environment (interpreter, OS and + every distribution in the isolated build environment with the dependency graph between them), plus the GitHub Actions + run when built there. (:issue:`3270`) + +Bugfixes - 21.9.0 +================= + +- Add ``metadata.timestamp`` (derived from ``SOURCE_DATE_EPOCH`` for reproducibility) and ``metadata.tools`` naming the + generator to the embedded SBOM, and validate both fields in the packaging checks that already run on every pull + request. (:issue:`3269`) +- Fix the embedded SBOM not conforming to the CycloneDX 1.6 schema (an invalid ``vendor`` key on the generator tool + entry) and validate it against the schema in CI. (:issue:`3270`) + +********************** + v21.8.1 (2026-09-19) +********************** + +Bugfixes - 21.8.1 +================= + +- Make the embedded SBOM's ``serialNumber`` a deterministic UUID derived from the package name, version and bundled + wheel hashes, so the same source tree produces a byte-identical SBOM, and validate the SBOM's structure as part of the + packaging checks that already run on every pull request. (:issue:`3268`) + +********************** + v21.8.0 (2026-09-19) +********************** + +Features - 21.8.0 +================= + +- Every published wheel now carries a `CycloneDX `_ SBOM at + ``.dist-info/sboms/virtualenv.cdx.json`` (:PEP:`770`), declaring the bundled ``pip`` and ``setuptools`` wheels that + generic SBOM tools cannot see on their own. GitHub attests it against the released sdist and wheel. (:issue:`3265`) + +*********************** + v21.7.16 (2026-09-19) +*********************** + +Bugfixes - 21.7.16 +================== + +- Fix ``activate.csh`` rendering the venv's prompt text wrong when it contains ``!`` (csh and tcsh both expand it) or + ``%`` (tcsh only, since plain csh has no ``%`` prompt escape at all). (:issue:`3263`) + +*********************** + v21.7.15 (2026-09-18) +*********************** + +Bugfixes - 21.7.15 +================== + +- Restore an empty ``PATH`` on ``deactivate`` in the bash and fish activators, and stop bash leaving the prompt in + ``PS1`` when it was unset before activation. (:issue:`3259`) +- Fix ``activate.csh`` failing to restore ``PATH`` and other saved variables on ``deactivate`` when ``PATH`` was already + empty. (:issue:`3260`) + +*********************** + v21.7.14 (2026-09-18) +*********************** + +Bugfixes - 21.7.14 +================== + +- Fix ``activate.csh`` failing with ``Event not found`` when the virtual environment path contains ``!``. + (:issue:`3256`) + +*********************** + v21.7.13 (2026-09-18) +*********************** + +Bugfixes - 21.7.13 +================== + +- Fix ``activate`` and ``activate.fish`` running commands embedded in the virtual environment path or in the + interpreter's Tcl/Tk library paths. (:issue:`3252`) + +*********************** + v21.7.12 (2026-09-18) +*********************** + +Bugfixes - 21.7.12 +================== + +- Fix ``activate.bat`` running arbitrary commands from a crafted ``--prompt``, ``VIRTUALENV_PROMPT``, or config file + value. (:issue:`3250`) +- Verify a downloaded seed wheel's sha256 against PyPI before seeding it into a virtual environment, skipped when a + custom pip index is configured. (:issue:`3251`) + +*********************** + v21.7.11 (2026-09-17) +*********************** + +Bugfixes - 21.7.11 +================== + +- Running ``activate.bat`` again before ``deactivate`` no longer makes ``deactivate`` leave the environment's + ``PKG_CONFIG_PATH``, ``TCL_LIBRARY`` and ``TK_LIBRARY`` behind, or lose values the user had set before the first + activation - by :user:`darrenhuai`. (:issue:`3245`) +- Write ``pyvenv.cfg`` values on a single line, so a prompt carrying a line boundary can no longer inject configuration. + ``--prompt``, the ``VIRTUALENV_PROMPT`` environment variable and the config file all set the prompt, and + ``pyvenv.cfg`` has no escape syntax, so a newline, a carriage return, or any other boundary ``str.splitlines`` + recognizes, such as ``U+2028``, started a new configuration line. Reading the file back picked up those lines as keys, + and since the last value for a key wins, they replaced anything written earlier, including ``home``. (:issue:`3247`) + +Improved Documentation - 21.7.11 +================================ + +- Document the policy for AI-assisted contributions and the licensing rules for dependencies. (:issue:`3239`) + +Misc - 21.7.11 +============== + +- :issue:`3238`, :issue:`3240`, :issue:`3241`, :issue:`3242`, :issue:`3243`, :issue:`3244`, :issue:`3246` + +*********************** + v21.7.10 (2026-09-15) +*********************** + +Bugfixes - 21.7.10 +================== + +- Skip blank lines, ``#`` comments and lines without ``=`` in ``pyvenv.cfg`` instead of raising ``ValueError`` - by + :user:`r3wretrhy`. (:issue:`3232`) +- ``deactivate`` in bash, fish and PowerShell unsets ``PKG_CONFIG_PATH`` when activation found it unset, instead of + keeping the environment's ``lib/pkgconfig``. csh activation no longer drops a ``PKG_CONFIG_PATH`` the user had set. + Activation in batch, fish, nushell and PowerShell no longer adds a trailing separator when ``PKG_CONFIG_PATH`` is + unset, and PowerShell and nushell build the value with the host's path separators - by :user:`darrenhuai`. + (:issue:`3233`) +- Activation in bash, csh, fish and PowerShell keeps the user's ``TCL_LIBRARY`` and ``TK_LIBRARY``, and ``deactivate`` + restores them. csh and PowerShell removed both variables on every activation, fish did so when the interpreter has + tcl, and bash kept the environment's value after ``deactivate`` when the variable was unset before - by + :user:`darrenhuai`. (:issue:`3234`) + +********************** + v21.7.9 (2026-09-09) +********************** + +Bugfixes - 21.7.9 +================= + +- Replace dangling symlinks, including interpreter aliases, when recreating an environment. This prevents + ``FileExistsError`` with ``--symlinks`` and writes outside the environment with ``--copies`` - by :user:`darrenhuai`. + (:issue:`3229`) +- Ignore malformed or unreadable ``virtualenv.ini`` files and report the error in the log and ``--help``. Accept a UTF-8 + byte order mark, as written by PowerShell 5 and older Notepad versions - by :user:`darrenhuai`. (:issue:`3230`) + +********************** + v21.7.8 (2026-09-01) +********************** + +Bugfixes - 21.7.8 +================= + +- A missing source needed by both install modes now disables the builtin creator through ``meta.error``. The check + compared ``RefWhen`` values against ``RefMust`` members, so creation went on to fail with a ``FileNotFoundError`` or a + dangling symlink - by :user:`darrenhuai`. (:issue:`3227`) + +********************** + v21.7.7 (2026-08-28) +********************** + +Bugfixes - 21.7.7 +================= + +- Bump the ``python-discovery`` minimum to ``>=1.6`` for ``PythonInfo.system_exe``, which reports the system interpreter + without the nullable typing of ``system_executable`` - by :user:`gaborbernat`. (:issue:`3224`) + +Improved Documentation - 21.7.7 +=============================== + +- Document the names the interpreter answers to inside a created environment. A new :doc:`reference/environment-layout` + page lists them per platform, the tutorial and the usage guide point at it, and the explanation of creators covers why + an environment carries several names and why Windows copies a redirector - by :user:`gaborbernat`. (:issue:`3225`) + +********************** + v21.7.6 (2026-08-28) +********************** + +Bugfixes - 21.7.6 +================= + +- On Windows, ``virtualenv`` no longer copies the CPython 3.13+ ``venvlauncher.exe`` shim into ``Scripts`` under the + shim's own name, and a host such as ``python_d.exe`` gets its alias back. The alias set took its names from the shim + that stands in for the interpreter, so every environment gained a stray launcher copy and lost the interpreter's own + file name - by :user:`darrenhuai`. (:issue:`3223`) + +********************** + v21.7.5 (2026-08-25) +********************** + +Bugfixes - 21.7.5 +================= + +- Fix the type check against ``python-discovery`` 1.5.2, whose annotations allow a ``None`` ``prefix`` and integer + ``sysconfig_vars`` values: config var substitution now skips a missing prefix and locating the shared libpython + requires string ``INSTSONAME``/``LIBDIR`` values. (:issue:`3211`) +- ``ExePathRef.can_run`` now checks the group and other execute bits instead of only the owner one, and returns + ``False`` rather than ``None`` when a file carries none of the three - by :user:`darrenhuai`. (:issue:`3217`) +- ``safe_delete`` no longer passes ``ignore_errors=True`` to ``shutil.rmtree``, which replaced its own chmod-and-retry + handler with a no-op and swallowed every failure. Read-only files - every file in a wheel image, which ``set_tree`` + marks - survived, so ``--reset-app-data`` and ``--clear`` kept trees they reported deleting. The handler now retries + only the deletion itself, keeps the other mode bits, and raises the original error for anything it cannot clear - by + :user:`darrenhuai`. (:issue:`3222`) + +********************** + v21.7.4 (2026-08-10) +********************** + +Bugfixes - 21.7.4 +================= + +- Upgrade embedded wheels: + + - setuptools to ``84.0.0`` from ``83.0.0`` (:issue:`3208`) + +********************** + v21.7.3 (2026-08-08) +********************** + +Bugfixes - 21.7.3 +================= + +- ``ListType.split_values`` now returns a ``list`` you can iterate more than once, and accepts ``bytes`` input instead + of raising ``TypeError`` - by :user:`darrenhuai`. (:issue:`3207`) + +********************** + v21.7.2 (2026-08-07) +********************** + +Bugfixes - 21.7.2 +================= + +- Upgrade embedded wheels: + + - pip to ``26.2.1`` from ``26.2`` (:issue:`3206`) + +********************** + v21.7.1 (2026-07-30) +********************** + +Bugfixes - 21.7.1 +================= + +- Upgrade embedded wheels: + + - pip to ``26.2`` from ``26.1.2`` (:issue:`u`) + +Improved Documentation - 21.7.1 +=============================== + +- Document that ``virtualenv.png`` is the official logo and ``virtualenv.svg`` is a simplified favicon-only mark - by + :user:`gaborbernat`. (:issue:`3199`) + +********************** + v21.7.0 (2026-07-21) +********************** + +Features - 21.7.0 +================= + +- Declare support for Python 3.15: CI now tests against the CPython 3.15 beta, including the free-threaded ``3.15t`` + build, and the ``Programming Language :: Python :: 3.15`` classifier is advertised. (:issue:`3192`) + +********************** + v21.6.1 (2026-07-10) +********************** + +Bugfixes - 21.6.1 +================= + +- Harden the fish activator prompt against user functions that shadow builtins. Routing ``functions``, ``printf``, + ``string``, ``echo`` and ``source``/``.`` through ``builtin`` stops a shadowing function (such as a dot-style + directory navigator that redefines ``.``) from hijacking the prompt and dropping the previous command's exit status, + matching the CPython fix in `gh-140006 `_. (:issue:`3185`) + +********************** + v21.6.0 (2026-07-06) +********************** + +Features - 21.6.0 +================= + +- Stop installing the ``_virtualenv.{py,pth}`` distutils import hook for Python 3.10 and later, where pip, setuptools + and CPython already ignore the install config keys it guards against; this removes the hook's startup import cost. The + hook is still installed for Python 3.9 - :issue:`3181`. (:issue:`3181`) + +********************** + v21.5.2 (2026-07-06) +********************** + +Bugfixes - 21.5.2 +================= + +- Upgrade embedded wheels: + + - setuptools to ``83.0.0`` (:issue:`3180`) + +********************** + v21.5.1 (2026-06-16) +********************** + +Bugfixes - 21.5.1 +================= + +- Refuse to create environments whose Python the bundled wheels no longer cover (currently below 3.9). virtualenv used + to substitute the newest bundled ``pip``, which cannot run on such a target, leaving a broken environment; seeder + selection now rejects it up front with a clear error. ``--no-seed`` and third-party seeders that ship compatible + wheels still work - by :user:`gaborbernat`. (:issue:`3171`) + +********************** + v21.5.0 (2026-06-13) +********************** + +Features - 21.5.0 +================= + +- Drop support for Python 3.8; virtualenv now requires Python 3.9 or later to run and to create environments. Remove the + embedded ``wheel`` seed package, which virtualenv bundled only for Python 3.8. The ``--wheel`` and ``--no-wheel`` + options stay as no-ops, but now warn that virtualenv will remove them in a release after 2026-12 - by + :user:`gaborbernat`. (:issue:`3170`) + +Bugfixes - 21.5.0 +================= + +- Upgrade embedded wheels: + + Removed wheel of ``0.47.0`` (:issue:`u`) + +********************** + v21.4.3 (2026-06-11) +********************** + +Bugfixes - 21.4.3 +================= + +- Upgrade embedded wheels: + + - pip to ``26.1.2`` from ``26.1.1`` (:issue:`u`) + +- Resolve executable-only symlinks when recording ``home`` and ``base-executable`` in ``pyvenv.cfg``, mirroring + CPython's ``getpath.realpath`` (python/cpython#115237), so environments created from a symlink to the interpreter + binary locate the base stdlib (for example python-build-standalone); a fully symlinked interpreter tree is kept as-is + - by :user:`gaborbernat`. (:issue:`3157`) +- Stop exporting ``PS1`` from the bash activator so child processes do not inherit shell prompt state. (:issue:`3158`) +- Handle CYGWIN/MSYS/MINGW path conversions in fish activation script - by user::`LuNoX`. (:issue:`3160`) + +********************** + v21.4.2 (2026-05-31) +********************** + +Bugfixes - 21.4.2 +================= + +- Stop ``deactivate`` in the bash/zsh activation script from aborting under ``set -e`` when ``hash -r`` fails (for + example with shell hashing disabled) by appending ``|| true``, matching CPython ``venv`` (gh-149701) and the existing + non-deactivate call - by :user:`gaborbernat`. (:issue:`3152`) + +********************** + v21.4.1 (2026-05-28) +********************** + +Bugfixes - 21.4.1 +================= + +- Fix Windows debug build ``venvlauncher_d.exe`` substitution never triggering because ``executables()`` compared the + source executable name instead of the target name, and fix ``AttributeError`` on ``debug_build`` attribute for + interpreter info objects missing the field - by :user:`gaborbernat`. (:issue:`3151`) + +********************** + v21.4.0 (2026-05-28) +********************** + +Features - 21.4.0 +================= + +- Remove dead code targeting Python versions below the supported target range (PyPy 3.6, deprecated importlib APIs) and + simplify the runtime import hook in ``_virtualenv.py`` - by :user:`gaborbernat`. (:issue:`3149`) +- Support Windows debug builds (``python_d.exe``, ``venvlauncher_d.exe``) matching CPython venv behavior, remove dead + ``__SCRIPT_DIR__`` replacement and ``has_shim`` version guard, drop unreachable Python 3.7 branch from + ``pyvenv_launch_patch_active``, and fix wheel deprecation message to say ``>= 3.9`` - by :user:`gaborbernat`. + (:issue:`3150`) + +********************** + v21.3.3 (2026-05-13) +********************** + +Bugfixes - 21.3.3 +================= + +- recognize GraalPy interpreters using the normalized ``GraalPy`` name - by :user:`timfel`. (:issue:`3144`) + +********************** + v21.3.2 (2026-05-12) +********************** + +No significant changes. + +********************** + v21.3.1 (2026-05-05) +********************** + +Bugfixes - 21.3.1 +================= + +- Upgrade embedded wheels: + + - pip to ``26.1.1`` from ``26.1`` (:issue:`3138`) + +********************** + v21.3.0 (2026-04-27) +********************** + +Features - 21.3.0 +================= + +- Re-introduce ``xonsh`` shell activator (``activate.xsh``) previously removed in 20.7.0, and make the plugin loader + prefer virtualenv's built-in entry points so a third-party package cannot override them by registering a duplicate + name. (:issue:`3003`) + +Bugfixes - 21.3.0 +================= + +- Upgrade embedded wheels: + + - pip to ``26.1`` (:issue:`3132`) + +********************** + v21.2.4 (2026-04-14) +********************** + +Bugfixes - 21.2.4 +================= + +- Security hardening: validate each entry of a seed wheel archive before extracting it so a tampered wheel cannot escape + the app-data image directory via an absolute path or ``..`` traversal. (:issue:`3118`) +- Security hardening: verify the SHA-256 of every bundled seed wheel when it is loaded so a corrupted or tampered file + on disk fails loud instead of being handed to pip. The hash table is generated alongside ``BUNDLE_SUPPORT`` by + ``tasks/upgrade_wheels.py``. (:issue:`3119`) +- Security hardening: validate the distribution name and version specifier passed to ``pip download`` when acquiring a + seed wheel so extras, pip flags, or shell metacharacters cannot be smuggled into the subprocess command line. + (:issue:`3120`) +- Security hardening: replace the string-prefix containment check in ``virtualenv.util.zipapp`` with + ``Path.relative_to`` so the zipapp extraction helpers refuse any path that does not resolve under the archive root. + (:issue:`3121`) +- Security hardening: do not silently fall back to an unverified HTTPS context when the periodic update request to PyPI + fails TLS verification. The returned metadata drives which wheel version virtualenv considers "up to date", so + accepting an unverified response lets a network-level attacker suppress security updates. Set + ``VIRTUALENV_PERIODIC_UPDATE_INSECURE=1`` to restore the previous behavior on hosts with broken trust stores. + (:issue:`3122`) + +********************** + v21.2.3 (2026-04-14) +********************** + +No significant changes. + +********************** + v21.2.2 (2026-04-13) +********************** + +Bugfixes - 21.2.2 +================= + +- Bump ``python-discovery`` minimum to ``>=1.2.2`` to include ``normalize_isa`` support - by :user:`rahuldevikar`. + (:issue:`3117`) + +********************** + v21.2.1 (2026-04-09) +********************** + +Bugfixes - 21.2.1 +================= + +- Upgrade embedded wheels: + + - setuptools to ``82.0.1`` from ``82.0.0`` (:issue:`3093`) + +- Use terminal width for help formatting instead of hardcoded 240. (:issue:`3110`) + +********************** + v21.2.0 (2026-03-09) +********************** + +Features - 21.2.0 +================= + +- Update embed wheel generator (``tasks/upgrade_wheels.py``) to include type annotations in generated output - by + :user:`rahuldevikar`. (:issue:`3075`) + +Bugfixes - 21.2.0 +================= + +- Pass ``--without-scm-ignore-files`` to subprocess venv on Python 3.13+ so virtualenv controls ``.gitignore`` creation, + fixing flaky ``test_create_no_seed`` and ``--no-vcs-ignore`` being ignored in subprocess path - by + :user:`gaborbernat`. (:issue:`3089`) +- Use ``BASH_SOURCE[0]`` instead of ``$0`` in the bash activate script relocation fallback, fixing incorrect ``PATH`` + when sourcing the activate script from a different directory - by :user:`gaborbernat`. (:issue:`3090`) + +********************** + v21.1.0 (2026-02-27) +********************** + +Features - 21.1.0 +================= + +- Add comprehensive type annotations across the entire codebase and ship a PEP 561 ``py.typed`` marker so downstream + consumers and type checkers recognize virtualenv as an inline-typed package - by :user:`rahuldevikar`. (:issue:`3075`) + +********************** + v21.0.0 (2026-02-25) +********************** + +Deprecations and Removals - 21.0.0 +================================== + +- The Python discovery logic has been extracted into a standalone ``python-discovery`` package on PyPI (`documentation + `_) and is now consumed as a dependency. If you previously imported + discovery internals directly (e.g. ``from virtualenv.discovery.py_info import PythonInfo``), switch to ``from + python_discovery import PythonInfo``. Backward-compatibility re-export shims are provided at + ``virtualenv.discovery.py_info``, ``virtualenv.discovery.py_spec``, and ``virtualenv.discovery.cached_py_info``, + however these are considered unsupported and may be removed in a future release - by :user:`gaborbernat`. + (:issue:`3070`) + +*********************** + v20.39.1 (2026-02-25) +*********************** + +Features - 20.39.1 +================== + +- Add support for creating virtual environments with RustPython - by :user:`elmjag`. (:issue:`3010`) + +*********************** + v20.39.0 (2026-02-23) +*********************** + +Features - 20.39.0 +================== + +- Automatically resolve version manager shims (pyenv, mise, asdf) to the real Python binary during discovery, preventing + incorrect interpreter selection when shims are on ``PATH`` - by :user:`gaborbernat`. (:issue:`3049`) +- Add architecture (ISA) awareness to Python discovery — users can now specify a CPU architecture suffix in the + ``--python`` spec string (e.g. ``cpython3.12-64-arm64``) to distinguish between interpreters that share the same + version and bitness but target different architectures. Uses ``sysconfig.get_platform()`` as the data source, with + cross-platform normalization (``amd64`` ↔ ``x86_64``, ``aarch64`` ↔ ``arm64``). Omitting the suffix preserves existing + behavior - by :user:`rahuldevikar`. (:issue:`3059`) + +*********************** + v20.38.0 (2026-02-19) +*********************** + +Features - 20.38.0 +================== + +- Store app data (pip/setuptools/wheel caches) under the OS cache directory (``platformdirs.user_cache_dir``) instead of + the data directory (``platformdirs.user_data_dir``). Existing app data at the old location is automatically migrated + on first use. This ensures cached files that can be redownloaded are placed in the standard cache location (e.g. + ``~/.cache`` on Linux, ``~/Library/Caches`` on macOS) where they are excluded from backups and can be cleaned by + system tools - by :user:`rahuldevikar`. (:issue:`1884`) (:issue:`1884`) +- Add ``PKG_CONFIG_PATH`` environment variable support to all activation scripts (Bash, Batch, PowerShell, Fish, C + Shell, Nushell, and Python). The virtualenv's ``lib/pkgconfig`` directory is now automatically prepended to + ``PKG_CONFIG_PATH`` on activation and restored on deactivation, enabling packages that use ``pkg-config`` during + build/install to find their configuration files - by :user:`rahuldevikar`. (:issue:`2637`) +- Upgrade embedded pip to ``26.0.1`` from ``25.3`` and setuptools to ``82.0.0``, ``75.3.4`` from ``75.3.2``, ``80.9.0`` + - by :user:`rahuldevikar`. (:issue:`3027`) +- Replace ``ty: ignore`` comments with proper type narrowing using assertions and explicit None checks - by + :user:`rahuldevikar`. (:issue:`3029`) + +Bugfixes - 20.38.0 +================== + +- Exclude pywin32 DLLs (``pywintypes*.dll``, ``pythoncom*.dll``) from being copied to the Scripts directory during + virtualenv creation on Windows. This fixes compatibility issues with pywin32, which expects its DLLs to be installed + in ``site-packages/pywin32_system32`` by its own post-install script - by :user:`rahuldevikar`. (:issue:`2662`) +- Preserve symlinks in ``pyvenv.cfg`` paths to match ``venv`` behavior. Use ``os.path.abspath()`` instead of + ``os.path.realpath()`` to normalize paths without resolving symlinks, fixing issues with Python installations accessed + via symlinked directories (common in network-mounted filesystems) - by :user:`rahuldevikar`. Fixes :issue:`2770`. + (:issue:`2770`) +- Fix Windows activation scripts to properly quote ``python.exe`` path, preventing failures when Python is installed in + a path with spaces (e.g., ``C:\Program Files``) and a file named ``C:\Program`` exists on the filesystem - by + :user:`rahuldevikar`. (:issue:`2985`) +- Fix ``bash -u`` (``set -o nounset``) compatibility in bash activation script by using ``${PKG_CONFIG_PATH:-}`` and + ``${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}}`` to handle unset ``PKG_CONFIG_PATH`` - by :user:`Fridayai700`. + (:issue:`3044`) +- Gracefully handle corrupted on-disk cache and invalid JSON from Python interrogation subprocess instead of crashing + with unhandled ``JSONDecodeError`` or ``KeyError`` - by :user:`gaborbernat`. (:issue:`3054`) + +*********************** + v20.36.1 (2026-01-09) +*********************** + +Bugfixes - 20.36.1 +================== + +- Fix TOCTOU vulnerabilities in app_data and lock directory creation that could be exploited via symlink attacks - + reported by :user:`tsigouris007`, fixed by :user:`gaborbernat`. (:issue:`3013`) + +*********************** + v20.36.0 (2026-01-07) +*********************** + +Features - 20.36.0 +================== + +- Add support for PEP 440 version specifiers in the ``--python`` flag. Users can now specify Python versions using + operators like ``>=``, ``<=``, ``~=``, etc. For example: ``virtualenv --python=">=3.12" myenv`` `. (:issue:`2994`) + +*********************** + v20.35.4 (2025-10-28) +*********************** Bugfixes - 20.35.4 -~~~~~~~~~~~~~~~~~~ -- Fix race condition in ``_virtualenv.py`` when file is overwritten during import, preventing ``NameError`` when ``_DISTUTILS_PATCH`` is accessed - by :user:`gracetyy`. (:issue:`2969`) +================== + +- Fix race condition in ``_virtualenv.py`` when file is overwritten during import, preventing ``NameError`` when + ``_DISTUTILS_PATCH`` is accessed - by :user:`gracetyy`. (:issue:`2969`) - Upgrade embedded wheels: - * pip to ``25.3`` from ``25.2`` (:issue:`2989`) + - pip to ``25.3`` from ``25.2`` (:issue:`2989`) -v20.35.3 (2025-10-10) ---------------------- +*********************** + v20.35.3 (2025-10-10) +*********************** Bugfixes - 20.35.3 -~~~~~~~~~~~~~~~~~~ +================== + - Accept RuntimeError in `test_too_many_open_files`, by :user:`esafak` (:issue:`2935`) -v20.35.2 (2025-10-10) ---------------------- +*********************** + v20.35.2 (2025-10-10) +*********************** Bugfixes - 20.35.2 -~~~~~~~~~~~~~~~~~~ +================== + - Revert out changes related to the extraction of the discovery module - by :user:`gaborbernat`. (:issue:`2978`) -v20.35.1 (2025-10-09) ---------------------- +*********************** + v20.35.1 (2025-10-09) +*********************** Bugfixes - 20.35.1 -~~~~~~~~~~~~~~~~~~ +================== + - Patch get_interpreter to handle missing cache and app_data - by :user:`esafak` (:issue:`2972`) - Fix backwards incompatible changes to ``PythonInfo`` - by :user:`gaborbernat`. (:issue:`2975`) -v20.35.0 (2025-10-08) ---------------------- +*********************** + v20.35.0 (2025-10-08) +*********************** Features - 20.35.0 -~~~~~~~~~~~~~~~~~~ +================== + - Add AppData and Cache protocols to discovery for decoupling - by :user:`esafak`. (:issue:`2074`) - Ensure python3.exe and python3 on Windows for Python 3 - by :user:`esafak`. (:issue:`2774`) Bugfixes - 20.35.0 -~~~~~~~~~~~~~~~~~~ +================== + - Replaced direct references to tcl/tk library paths with getattr - by :user:`esafak` (:issue:`2944`) - Restore absolute import of fs_is_case_sensitive - by :user:`esafak`. (:issue:`2955`) -v20.34.0 (2025-08-13) ---------------------- +*********************** + v20.34.0 (2025-08-13) +*********************** Features - 20.34.0 -~~~~~~~~~~~~~~~~~~ -- Abstract out caching in discovery - by :user:`esafak`. - Decouple `FileCache` from `py_info` (discovery) - by :user:`esafak`. - Remove references to py_info in FileCache - by :user:`esafak`. - Decouple discovery from creator plugins - by :user:`esafak`. - Decouple discovery by duplicating info utils - by :user:`esafak`. (:issue:`2074`) +================== + +- Abstract out caching in discovery - by :user:`esafak`. Decouple `FileCache` from `py_info` (discovery) - by + :user:`esafak`. Remove references to py_info in FileCache - by :user:`esafak`. Decouple discovery from creator plugins + - by :user:`esafak`. Decouple discovery by duplicating info utils - by :user:`esafak`. (:issue:`2074`) - Add PyPy 3.11 support. Contributed by :user:`esafak`. (:issue:`2932`) Bugfixes - 20.34.0 -~~~~~~~~~~~~~~~~~~ +================== + - Upgrade embedded wheel pip to ``25.2`` from ``25.1.1`` - by :user:`gaborbernat`. (:issue:`2333`) - Accept RuntimeError in `test_too_many_open_files`, by :user:`esafak` (:issue:`2935`) - Python in PATH takes precedence over uv-managed python. Contributed by :user:`edgarrmondragon`. (:issue:`2952`) -v20.33.1 (2025-08-05) ---------------------- +*********************** + v20.33.1 (2025-08-05) +*********************** Bugfixes - 20.33.1 -~~~~~~~~~~~~~~~~~~ -- Correctly unpack _get_tcl_tk_libs() response in PythonInfo. - Contributed by :user:`esafak`. (:issue:`2930`) -- Restore `py_info.py` timestamp in `test_py_info_cache_invalidation_on_py_info_change` - Contributed by :user:`esafak`. (:issue:`2933`) +================== -v20.33.0 (2025-08-03) ---------------------- +- Correctly unpack _get_tcl_tk_libs() response in PythonInfo. Contributed by :user:`esafak`. (:issue:`2930`) +- Restore `py_info.py` timestamp in `test_py_info_cache_invalidation_on_py_info_change` Contributed by :user:`esafak`. + (:issue:`2933`) + +*********************** + v20.33.0 (2025-08-03) +*********************** Features - 20.33.0 -~~~~~~~~~~~~~~~~~~ -- Added support for Tcl and Tkinter. You're welcome. - Contributed by :user:`esafak`. (:issue:`425`) +================== + +- Added support for Tcl and Tkinter. You're welcome. Contributed by :user:`esafak`. (:issue:`425`) Bugfixes - 20.33.0 -~~~~~~~~~~~~~~~~~~ -- Prevent logging setup when --help is passed, fixing a flaky test. - Contributed by :user:`esafak`. (:issue:`u`) -- Fix cache invalidation for PythonInfo by hashing `py_info.py`. - Contributed by :user:`esafak`. (:issue:`2467`) -- When no discovery plugins are found, the application would crash with a StopIteration. - This change catches the StopIteration and raises a RuntimeError with a more informative message. - Contributed by :user:`esafak`. (:issue:`2493`) -- Stop `--try-first-with` overriding absolute `--python` paths. - Contributed by :user:`esafak`. (:issue:`2659`) -- Force UTF-8 encoding for pip download - Contributed by :user:`esafak`. (:issue:`2780`) -- Creating a virtual environment on a filesystem without symlink-support would fail even with `--copies` - Make `fs_supports_symlink` perform a real symlink creation check on all platforms. - Contributed by :user:`esafak`. (:issue:`2786`) +================== + +- Prevent logging setup when --help is passed, fixing a flaky test. Contributed by :user:`esafak`. (:issue:`u`) +- Fix cache invalidation for PythonInfo by hashing `py_info.py`. Contributed by :user:`esafak`. (:issue:`2467`) +- When no discovery plugins are found, the application would crash with a StopIteration. This change catches the + StopIteration and raises a RuntimeError with a more informative message. Contributed by :user:`esafak`. + (:issue:`2493`) +- Stop `--try-first-with` overriding absolute `--python` paths. Contributed by :user:`esafak`. (:issue:`2659`) +- Force UTF-8 encoding for pip download Contributed by :user:`esafak`. (:issue:`2780`) +- Creating a virtual environment on a filesystem without symlink-support would fail even with `--copies` Make + `fs_supports_symlink` perform a real symlink creation check on all platforms. Contributed by :user:`esafak`. + (:issue:`2786`) - Add a note to the user guide recommending the use of a specific Python version when creating virtual environments. Contributed by :user:`esafak`. (:issue:`2808`) -- Fix 'Too many open files' error due to a file descriptor leak in virtualenv's locking mechanism. - Contributed by :user:`esafak`. (:issue:`2834`) -- Support renamed Windows venv redirector (`venvlauncher.exe` and `venvwlauncher.exe`) on Python 3.13 - Contributed by :user:`esafak`. (:issue:`2851`) +- Fix 'Too many open files' error due to a file descriptor leak in virtualenv's locking mechanism. Contributed by + :user:`esafak`. (:issue:`2834`) +- Support renamed Windows venv redirector (`venvlauncher.exe` and `venvwlauncher.exe`) on Python 3.13 Contributed by + :user:`esafak`. (:issue:`2851`) - Resolve Nushell activation script deprecation warnings by dynamically selecting the ``--optional`` flag for Nushell - ``get`` command on version 0.106.0 and newer, while retaining the deprecated ``-i`` flag for older versions to maintain - compatibility. Contributed by :user:`gaborbernat`. (:issue:`2910`) + ``get`` command on version 0.106.0 and newer, while retaining the deprecated ``-i`` flag for older versions to + maintain compatibility. Contributed by :user:`gaborbernat`. (:issue:`2910`) -v20.32.0 (2025-07-20) ---------------------- +*********************** + v20.32.0 (2025-07-20) +*********************** Features - 20.32.0 -~~~~~~~~~~~~~~~~~~ +================== + - Warn on incorrect invocation of Nushell activation script - by :user:`esafak`. (:issue:`nushell_activation`) - Discover uv-managed Python installations (:issue:`2901`) Bugfixes - 20.32.0 -~~~~~~~~~~~~~~~~~~ +================== + - Ignore missing absolute paths for python discovery - by :user:`esafak` (:issue:`2870`) - Upgrade embedded setuptools to ``80.9.0`` from ``80.3.1`` - by :user:`gaborbernat`. (:issue:`2900`) -v20.31.2 (2025-05-08) ---------------------- +*********************** + v20.31.2 (2025-05-08) +*********************** No significant changes. - -v20.31.1 (2025-05-05) ---------------------- +*********************** + v20.31.1 (2025-05-05) +*********************** Bugfixes - 20.31.1 -~~~~~~~~~~~~~~~~~~ +================== + - Upgrade embedded wheels: - * pip to ``25.1.1`` from ``25.1`` - * setuptools to ``80.3.1`` from ``78.1.0`` (:issue:`2880`) + - pip to ``25.1.1`` from ``25.1`` + - setuptools to ``80.3.1`` from ``78.1.0`` (:issue:`2880`) -v20.31.0 (2025-05-05) ---------------------- +*********************** + v20.31.0 (2025-05-05) +*********************** Features - 20.31.0 -~~~~~~~~~~~~~~~~~~ -- No longer bundle ``wheel`` wheels (except on Python 3.8), ``setuptools`` includes native ``bdist_wheel`` support. Update ``pip`` to ``25.1``. (:issue:`2868`) +================== + +- No longer bundle ``wheel`` wheels (except on Python 3.8), ``setuptools`` includes native ``bdist_wheel`` support. + Update ``pip`` to ``25.1``. (:issue:`2868`) Bugfixes - 20.31.0 -~~~~~~~~~~~~~~~~~~ -- ``get_embed_wheel()`` no longer fails with a :exc:`TypeError` when it is - called with an unknown *distribution*. (:issue:`2877`) +================== + +- ``get_embed_wheel()`` no longer fails with a :exc:`TypeError` when it is called with an unknown *distribution*. + (:issue:`2877`) - Fix ``HelpFormatter`` error with Python 3.14.0b1. (:issue:`2878`) -v20.30.0 (2025-03-31) ---------------------- +*********************** + v20.30.0 (2025-03-31) +*********************** Features - 20.30.0 -~~~~~~~~~~~~~~~~~~ +================== + - Add support for `GraalPy `_. (:issue:`2832`) Bugfixes - 20.30.0 -~~~~~~~~~~~~~~~~~~ +================== + - Upgrade embedded wheels: - * setuptools to ``78.1.0`` from ``75.3.2`` (:issue:`2863`) + - setuptools to ``78.1.0`` from ``75.3.2`` (:issue:`2863`) -v20.29.3 (2025-03-06) ---------------------- +*********************** + v20.29.3 (2025-03-06) +*********************** Bugfixes - 20.29.3 -~~~~~~~~~~~~~~~~~~ +================== + - Ignore unreadable directories in ``PATH``. (:issue:`2794`) -v20.29.2 (2025-02-10) ---------------------- +*********************** + v20.29.2 (2025-02-10) +*********************** Bugfixes - 20.29.2 -~~~~~~~~~~~~~~~~~~ +================== + - Remove old virtualenv wheel from the source distribution - by :user:`gaborbernat`. (:issue:`2841`) - Upgrade embedded wheel pip to ``25.0.1`` from ``24.3.1`` - by :user:`gaborbernat`. (:issue:`2843`) -v20.29.1 (2025-01-17) ---------------------- +*********************** + v20.29.1 (2025-01-17) +*********************** Bugfixes - 20.29.1 -~~~~~~~~~~~~~~~~~~ +================== + - Fix PyInfo cache incompatibility warnings - by :user:`robsdedude`. (:issue:`2827`) -v20.29.0 (2025-01-15) ---------------------- +*********************** + v20.29.0 (2025-01-15) +*********************** Features - 20.29.0 -~~~~~~~~~~~~~~~~~~ +================== + - Add support for selecting free-threaded Python interpreters, e.g., `python3.13t`. (:issue:`2809`) Bugfixes - 20.29.0 -~~~~~~~~~~~~~~~~~~ +================== + - Upgrade embedded wheels: - * setuptools to ``75.8.0`` from ``75.6.0`` (:issue:`2823`) + - setuptools to ``75.8.0`` from ``75.6.0`` (:issue:`2823`) -v20.28.1 (2025-01-02) ---------------------- +*********************** + v20.28.1 (2025-01-02) +*********************** Bugfixes - 20.28.1 -~~~~~~~~~~~~~~~~~~ +================== + - Skip tcsh tests on broken tcsh versions - by :user:`gaborbernat`. (:issue:`2814`) -v20.28.0 (2024-11-25) ---------------------- +*********************** + v20.28.0 (2024-11-25) +*********************** Features - 20.28.0 -~~~~~~~~~~~~~~~~~~ +================== + - Write CACHEDIR.TAG file on creation - by "user:`neilramsay`. (:issue:`2803`) -v20.27.2 (2024-11-25) ---------------------- +*********************** + v20.27.2 (2024-11-25) +*********************** Bugfixes - 20.27.2 -~~~~~~~~~~~~~~~~~~ +================== + - Upgrade embedded wheels: - * setuptools to ``75.3.0`` from ``75.2.0`` (:issue:`2798`) + - setuptools to ``75.3.0`` from ``75.2.0`` (:issue:`2798`) + - Upgrade embedded wheels: - * wheel to ``0.45.0`` from ``0.44.0`` - * setuptools to ``75.5.0`` (:issue:`2800`) + - wheel to ``0.45.0`` from ``0.44.0`` + - setuptools to ``75.5.0`` (:issue:`2800`) + - no longer forcibly echo off during windows batch activation (:issue:`2801`) - Upgrade embedded wheels: - * setuptools to ``75.6.0`` from ``75.5.0`` - * wheel to ``0.45.1`` from ``0.45.0`` (:issue:`2804`) + - setuptools to ``75.6.0`` from ``75.5.0`` + - wheel to ``0.45.1`` from ``0.45.0`` (:issue:`2804`) -v20.27.1 (2024-10-28) ---------------------- +*********************** + v20.27.1 (2024-10-28) +*********************** Bugfixes - 20.27.1 -~~~~~~~~~~~~~~~~~~ +================== + - Upgrade embedded wheels: - * pip to ``24.3.1`` from ``24.2`` (:issue:`2789`) + - pip to ``24.3.1`` from ``24.2`` (:issue:`2789`) -v20.27.0 (2024-10-17) ---------------------- +*********************** + v20.27.0 (2024-10-17) +*********************** Features - 20.27.0 -~~~~~~~~~~~~~~~~~~ +================== + - Drop 3.7 support as the CI environments no longer allow it running - by :user:`gaborbernat`. (:issue:`2758`) Bugfixes - 20.27.0 -~~~~~~~~~~~~~~~~~~ -- When a ``$PATH`` entry cannot be checked for existence, skip it instead of terminating - by :user:`hroncok`. (:issue:`2782`) -- Upgrade embedded wheels: +================== - * setuptools to ``75.2.0`` from ``75.1.0`` - * Removed pip of ``24.0`` - * Removed setuptools of ``68.0.0`` - * Removed wheel of ``0.42.0`` +- When a ``$PATH`` entry cannot be checked for existence, skip it instead of terminating - by :user:`hroncok`. + (:issue:`2782`) +- Upgrade embedded wheels: + - setuptools to ``75.2.0`` from ``75.1.0`` + - Removed pip of ``24.0`` + - Removed setuptools of ``68.0.0`` + - Removed wheel of ``0.42.0`` - by :user:`gaborbernat`. (:issue:`2783`) + - Fix zipapp is broken on Windows post distlib ``0.3.9`` - by :user:`gaborbernat`. (:issue:`2784`) -v20.26.6 (2024-09-27) ---------------------- +*********************** + v20.26.6 (2024-09-27) +*********************** Bugfixes - 20.26.6 -~~~~~~~~~~~~~~~~~~ -- Properly quote string placeholders in activation script templates to mitigate - potential command injection - by :user:`y5c4l3`. (:issue:`2768`) +================== -v20.26.5 (2024-09-17) ---------------------- +- Properly quote string placeholders in activation script templates to mitigate potential command injection - by + :user:`y5c4l3`. (:issue:`2768`) + +*********************** + v20.26.5 (2024-09-17) +*********************** Bugfixes - 20.26.5 -~~~~~~~~~~~~~~~~~~ +================== + - Upgrade embedded wheels: setuptools to ``75.1.0`` from ``74.1.2`` - by :user:`gaborbernat`. (:issue:`2765`) -v20.26.4 (2024-09-07) ---------------------- +*********************** + v20.26.4 (2024-09-07) +*********************** Bugfixes - 20.26.4 -~~~~~~~~~~~~~~~~~~ +================== + - no longer create `()` output in console during activation of a virtualenv by .bat file. (:issue:`2728`) - Upgrade embedded wheels: - * wheel to ``0.44.0`` from ``0.43.0`` - * pip to ``24.2`` from ``24.1`` - * setuptools to ``74.1.2`` from ``70.1.0`` (:issue:`2760`) + - wheel to ``0.44.0`` from ``0.43.0`` + - pip to ``24.2`` from ``24.1`` + - setuptools to ``74.1.2`` from ``70.1.0`` (:issue:`2760`) -v20.26.3 (2024-06-21) ---------------------- +*********************** + v20.26.3 (2024-06-21) +*********************** Bugfixes - 20.26.3 -~~~~~~~~~~~~~~~~~~ +================== + - Upgrade embedded wheels: - * setuptools to ``70.1.0`` from ``69.5.1`` - * pip to ``24.1`` from ``24.0`` (:issue:`2741`) + - setuptools to ``70.1.0`` from ``69.5.1`` + - pip to ``24.1`` from ``24.0`` (:issue:`2741`) -v20.26.2 (2024-05-13) ---------------------- +*********************** + v20.26.2 (2024-05-13) +*********************** Bugfixes - 20.26.2 -~~~~~~~~~~~~~~~~~~ -- ``virtualenv.pyz`` no longer fails when zipapp path contains a symlink - by :user:`HandSonic` and :user:`petamas`. (:issue:`1949`) +================== + +- ``virtualenv.pyz`` no longer fails when zipapp path contains a symlink - by :user:`HandSonic` and :user:`petamas`. + (:issue:`1949`) - Fix bad return code from activate.sh if hashing is disabled - by :user:'fenkes-ibm'. (:issue:`2717`) -v20.26.1 (2024-04-29) ---------------------- +*********************** + v20.26.1 (2024-04-29) +*********************** Bugfixes - 20.26.1 -~~~~~~~~~~~~~~~~~~ +================== + - fix PATH-based Python discovery on Windows - by :user:`ofek`. (:issue:`2712`) -v20.26.0 (2024-04-23) ---------------------- +*********************** + v20.26.0 (2024-04-23) +*********************** Bugfixes - 20.26.0 -~~~~~~~~~~~~~~~~~~ -- allow builtin discovery to discover specific interpreters (e.g. ``python3.12``) given an unspecific spec (e.g. ``python3``) - by :user:`flying-sheep`. (:issue:`2709`) +================== + +- allow builtin discovery to discover specific interpreters (e.g. ``python3.12``) given an unspecific spec (e.g. + ``python3``) - by :user:`flying-sheep`. (:issue:`2709`) -v20.25.3 (2024-04-17) ---------------------- +*********************** + v20.25.3 (2024-04-17) +*********************** Bugfixes - 20.25.3 -~~~~~~~~~~~~~~~~~~ +================== + - Python 3.13.0a6 renamed pathmod to parser. (:issue:`2702`) -v20.25.2 (2024-04-16) ---------------------- +*********************** + v20.25.2 (2024-04-16) +*********************** Bugfixes - 20.25.2 -~~~~~~~~~~~~~~~~~~ +================== + - Upgrade embedded wheels: - setuptools of ``69.1.0`` to ``69.5.1`` - wheel of ``0.42.0`` to ``0.43.0`` (:issue:`2699`) -v20.25.1 (2024-02-21) ---------------------- +*********************** + v20.25.1 (2024-02-21) +*********************** Bugfixes - 20.25.1 -~~~~~~~~~~~~~~~~~~ +================== + - Upgrade embedded wheels: - * setuptools to ``69.0.3`` from ``69.0.2`` - * pip to ``23.3.2`` from ``23.3.1`` (:issue:`2681`) + - setuptools to ``69.0.3`` from ``69.0.2`` + - pip to ``23.3.2`` from ``23.3.1`` (:issue:`2681`) + - Upgrade embedded wheels: - pip ``23.3.2`` to ``24.0``, - setuptools ``69.0.3`` to ``69.1.0``. (:issue:`2691`) Misc - 20.25.1 -~~~~~~~~~~~~~~ +============== + - :issue:`2688` -v20.25.0 (2023-12-01) ---------------------- +*********************** + v20.25.0 (2023-12-01) +*********************** Features - 20.25.0 -~~~~~~~~~~~~~~~~~~ +================== + - The tests now pass on the CI with Python 3.13.0a2 - by :user:`hroncok`. (:issue:`2673`) Bugfixes - 20.25.0 -~~~~~~~~~~~~~~~~~~ +================== + - Upgrade embedded wheels: - * wheel to ``0.41.3`` from ``0.41.2`` (:issue:`2665`) + - wheel to ``0.41.3`` from ``0.41.2`` (:issue:`2665`) + - Upgrade embedded wheels: - * wheel to ``0.42.0`` from ``0.41.3`` - * setuptools to ``69.0.2`` from ``68.2.2`` (:issue:`2669`) + - wheel to ``0.42.0`` from ``0.41.3`` + - setuptools to ``69.0.2`` from ``68.2.2`` (:issue:`2669`) -v20.24.6 (2023-10-23) ---------------------- +*********************** + v20.24.6 (2023-10-23) +*********************** Bugfixes - 20.24.6 -~~~~~~~~~~~~~~~~~~ +================== + - Use get_hookimpls method instead of the private attribute in tests. (:issue:`2649`) - Upgrade embedded wheels: - * setuptools to ``68.2.2`` from ``68.2.0`` - * pip to ``23.3.1`` from ``23.2.1`` (:issue:`2656`) - + - setuptools to ``68.2.2`` from ``68.2.0`` + - pip to ``23.3.1`` from ``23.2.1`` (:issue:`2656`) -v20.24.5 (2023-09-08) ---------------------- +*********************** + v20.24.5 (2023-09-08) +*********************** Bugfixes - 20.24.5 -~~~~~~~~~~~~~~~~~~ +================== + - Declare PyPy 3.10 support - by :user:`cclauss`. (:issue:`2638`) - Brew on macOS no longer allows copy builds - disallow choosing this by :user:`gaborbernat`. (:issue:`2640`) - Upgrade embedded wheels: - * setuptools to ``68.2.0`` from ``68.1.2`` (:issue:`2642`) + - setuptools to ``68.2.0`` from ``68.1.2`` (:issue:`2642`) - -v20.24.4 (2023-08-30) ---------------------- +*********************** + v20.24.4 (2023-08-30) +*********************** Bugfixes - 20.24.4 -~~~~~~~~~~~~~~~~~~ -- Upgrade embedded wheels: +================== - * setuptools to ``68.1.2`` from ``68.1.0`` on ``3.8+`` - * wheel to ``0.41.2`` from ``0.41.1`` on ``3.7+`` (:issue:`2628`) +- Upgrade embedded wheels: + - setuptools to ``68.1.2`` from ``68.1.0`` on ``3.8+`` + - wheel to ``0.41.2`` from ``0.41.1`` on ``3.7+`` (:issue:`2628`) -v20.24.3 (2023-08-11) ---------------------- +*********************** + v20.24.3 (2023-08-11) +*********************** Bugfixes - 20.24.3 -~~~~~~~~~~~~~~~~~~ +================== + - Fixed ResourceWarning on exit caused by periodic update subprocess (:issue:`2472`) - Upgrade embedded wheels: - * wheel to ``0.41.1`` from ``0.41.0`` (:issue:`2622`) + - wheel to ``0.41.1`` from ``0.41.0`` (:issue:`2622`) Misc - 20.24.3 -~~~~~~~~~~~~~~ -- :issue:`2610` +============== +- :issue:`2610` -v20.24.2 (2023-07-24) ---------------------- +*********************** + v20.24.2 (2023-07-24) +*********************** Bugfixes - 20.24.2 -~~~~~~~~~~~~~~~~~~ -- Upgrade embedded wheels: +================== - * pip to ``23.2.1`` from ``23.2`` - * wheel to ``0.41.0`` from ``0.40.0`` (:issue:`2614`) +- Upgrade embedded wheels: + - pip to ``23.2.1`` from ``23.2`` + - wheel to ``0.41.0`` from ``0.40.0`` (:issue:`2614`) -v20.24.1 (2023-07-19) ---------------------- +*********************** + v20.24.1 (2023-07-19) +*********************** Bugfixes - 20.24.1 -~~~~~~~~~~~~~~~~~~ -- Upgrade embedded wheels: +================== - * pip to ``23.2`` from ``23.1.2`` - by :user:`arielkirkwood` (:issue:`2611`) +- Upgrade embedded wheels: + - pip to ``23.2`` from ``23.1.2`` - by :user:`arielkirkwood` (:issue:`2611`) -v20.24.0 (2023-07-14) ---------------------- +*********************** + v20.24.0 (2023-07-14) +*********************** Features - 20.24.0 -~~~~~~~~~~~~~~~~~~ -- Export the prompt prefix as ``VIRTUAL_ENV_PROMPT`` when activating a virtual - environment - by :user:`jimporter`. (:issue:`2194`) +================== + +- Export the prompt prefix as ``VIRTUAL_ENV_PROMPT`` when activating a virtual environment - by :user:`jimporter`. + (:issue:`2194`) Bugfixes - 20.24.0 -~~~~~~~~~~~~~~~~~~ +================== + - Fix test suite - by :user:`gaborbernat`. (:issue:`2592`) - Upgrade embedded wheels: - * setuptools to ``68.0.0`` from ``67.8.0`` (:issue:`2607`) + - setuptools to ``68.0.0`` from ``67.8.0`` (:issue:`2607`) - -v20.23.1 (2023-06-16) ---------------------- +*********************** + v20.23.1 (2023-06-16) +*********************** Bugfixes - 20.23.1 -~~~~~~~~~~~~~~~~~~ -- update and simplify nushell activation script, fixes an issue on Windows resulting in consecutive command not found - by :user:`melMass`. (:issue:`2572`) -- Upgrade embedded wheels: +================== - * setuptools to ``67.8.0`` from ``67.7.2`` (:issue:`2588`) +- update and simplify nushell activation script, fixes an issue on Windows resulting in consecutive command not found - + by :user:`melMass`. (:issue:`2572`) +- Upgrade embedded wheels: + - setuptools to ``67.8.0`` from ``67.7.2`` (:issue:`2588`) -v20.23.0 (2023-04-27) ---------------------- +*********************** + v20.23.0 (2023-04-27) +*********************** Features - 20.23.0 -~~~~~~~~~~~~~~~~~~ +================== + - Do not install ``wheel`` and ``setuptools`` seed packages for Python 3.12+. To restore the old behavior use: - for ``wheel`` use ``VIRTUALENV_WHEEL=bundle`` environment variable or ``--wheel=bundle`` CLI flag, - for ``setuptools`` use ``VIRTUALENV_SETUPTOOLS=bundle`` environment variable or ``--setuptools=bundle`` CLI flag. By :user:`chrysle`. (:issue:`2487`) + - 3.12 support - by :user:`gaborbernat`. (:issue:`2558`) Bugfixes - 20.23.0 -~~~~~~~~~~~~~~~~~~ -- Prevent ``PermissionError`` when using venv creator on systems that deliver files without user write - permission - by :user:`kulikjak`. (:issue:`2543`) -- Upgrade setuptools to ``67.7.2`` from ``67.6.1`` and pip to ``23.1.2`` from ``23.1`` - by :user:`szleb`. (:issue:`2560`) +================== +- Prevent ``PermissionError`` when using venv creator on systems that deliver files without user write permission - by + :user:`kulikjak`. (:issue:`2543`) +- Upgrade setuptools to ``67.7.2`` from ``67.6.1`` and pip to ``23.1.2`` from ``23.1`` - by :user:`szleb`. + (:issue:`2560`) -v20.22.0 (2023-04-19) ---------------------- +*********************** + v20.22.0 (2023-04-19) +*********************** Features - 20.22.0 -~~~~~~~~~~~~~~~~~~ +================== + - Drop support for creating Python <=3.6 (including 2) interpreters. Removed pip of ``20.3.4``, ``21.3.1``; wheel of ``0.37.1``; setuptools of ``59.6.0``, ``44.1.1``, ``50.3.2``- by :user:`gaborbernat`. (:issue:`2548`) - -v20.21.1 (2023-04-19) ---------------------- +*********************** + v20.21.1 (2023-04-19) +*********************** Bugfixes - 20.21.1 -~~~~~~~~~~~~~~~~~~ +================== + - Add ``tox.ini`` to sdist - by :user:`mtelka`. (:issue:`2511`) - Move the use of 'let' in nushell to ensure compatibility with future releases of nushell, where 'let' no longer assumes that its initializer is a full expressions. (:issue:`2527`) -- The nushell command 'str collect' has been superseded by the 'str join' command. The activate.nu script has - been updated to reflect this change. (:issue:`2532`) +- The nushell command 'str collect' has been superseded by the 'str join' command. The activate.nu script has been + updated to reflect this change. (:issue:`2532`) - Upgrade embedded wheels: - * wheel to ``0.40.0`` from ``0.38.4`` - * setuptools to ``67.6.1`` from ``67.4.0`` - * pip to ``23.1`` from ``23.0.1`` (:issue:`2546`) + - wheel to ``0.40.0`` from ``0.38.4`` + - setuptools to ``67.6.1`` from ``67.4.0`` + - pip to ``23.1`` from ``23.0.1`` (:issue:`2546`) - -v20.21.0 (2023-03-12) ---------------------- +*********************** + v20.21.0 (2023-03-12) +*********************** Features - 20.21.0 -~~~~~~~~~~~~~~~~~~ +================== + - Make closure syntax explicitly starts with {||. (:issue:`2512`) Bugfixes - 20.21.0 -~~~~~~~~~~~~~~~~~~ -- Add ``print`` command to nushell print_prompt to ensure compatibility with future release of nushell, - where intermediate commands no longer print their result to stdout. (:issue:`2514`) -- Do not assume the default encoding. (:issue:`2515`) -- Make ``ReentrantFileLock`` thread-safe and, - thereby, fix race condition in ``virtualenv.cli_run`` - by :user:`radoering`. (:issue:`2516`) +================== +- Add ``print`` command to nushell print_prompt to ensure compatibility with future release of nushell, where + intermediate commands no longer print their result to stdout. (:issue:`2514`) +- Do not assume the default encoding. (:issue:`2515`) +- Make ``ReentrantFileLock`` thread-safe and, thereby, fix race condition in ``virtualenv.cli_run`` - by + :user:`radoering`. (:issue:`2516`) -v20.20.0 (2023-02-28) ---------------------- +*********************** + v20.20.0 (2023-02-28) +*********************** Features - 20.20.0 -~~~~~~~~~~~~~~~~~~ -- Change environment variable existence check in Nushell activation script to not use deprecated command. (:issue:`2506`) +================== + +- Change environment variable existence check in Nushell activation script to not use deprecated command. + (:issue:`2506`) Bugfixes - 20.20.0 -~~~~~~~~~~~~~~~~~~ -- Discover CPython implementations distributed on Windows by any organization - by :user:`faph`. (:issue:`2504`) -- Upgrade embedded setuptools to ``67.4.0`` from ``67.1.0`` and pip to ``23.0.1`` from ``23.0`` - by :user:`gaborbernat`. (:issue:`2510`) +================== +- Discover CPython implementations distributed on Windows by any organization - by :user:`faph`. (:issue:`2504`) +- Upgrade embedded setuptools to ``67.4.0`` from ``67.1.0`` and pip to ``23.0.1`` from ``23.0`` - by + :user:`gaborbernat`. (:issue:`2510`) -v20.19.0 (2023-02-07) ---------------------- +*********************** + v20.19.0 (2023-02-07) +*********************** Features - 20.19.0 -~~~~~~~~~~~~~~~~~~ -- Allow platformdirs version ``3`` - by :user:`cdce8p`. (:issue:`2499`) +================== +- Allow platformdirs version ``3`` - by :user:`cdce8p`. (:issue:`2499`) -v20.18.0 (2023-02-06) ---------------------- +*********************** + v20.18.0 (2023-02-06) +*********************** Features - 20.18.0 -~~~~~~~~~~~~~~~~~~ +================== + - Drop ``3.6`` runtime support (can still create ``2.7+``) - by :user:`gaborbernat`. (:issue:`2489`) Bugfixes - 20.18.0 -~~~~~~~~~~~~~~~~~~ +================== + - Fix broken prompt in Nushell when activating virtual environment - by :user:`kubouc`. (:issue:`2481`) - Bump embedded pip to ``23.0`` and setuptools to ``67.1`` - by :user:`gaborbernat`. (:issue:`2489`) - -v20.17.1 (2022-12-05) ---------------------- +*********************** + v20.17.1 (2022-12-05) +*********************** Bugfixes - 20.17.1 -~~~~~~~~~~~~~~~~~~ -- A ``py`` or ``python`` spec means any Python rather than ``CPython`` - by :user:`gaborbernat`. (`#2460 `_) -- Make ``activate.nu`` respect ``VIRTUAL_ENV_DISABLE_PROMPT`` and not set the prompt if requested - by :user:`m-lima`. (`#2461 `_) +================== +- A ``py`` or ``python`` spec means any Python rather than ``CPython`` - by :user:`gaborbernat`. (`#2460 + `_) +- Make ``activate.nu`` respect ``VIRTUAL_ENV_DISABLE_PROMPT`` and not set the prompt if requested - by :user:`m-lima`. + (`#2461 `_) -v20.17.0 (2022-11-27) ---------------------- +*********************** + v20.17.0 (2022-11-27) +*********************** Features - 20.17.0 -~~~~~~~~~~~~~~~~~~ -- Change Nushell activation script to be a module meant to be activated as an overlay. (`#2422 `_) -- Update operator used in Nushell activation script to be compatible with future versions. (`#2450 `_) +================== + +- Change Nushell activation script to be a module meant to be activated as an overlay. (`#2422 + `_) +- Update operator used in Nushell activation script to be compatible with future versions. (`#2450 + `_) Bugfixes - 20.17.0 -~~~~~~~~~~~~~~~~~~ -- Do not use deprecated API from ``importlib.resources`` on Python 3.10 or later - by :user:`gaborbernat`. (`#2448 `_) -- Upgrade embedded setuptools to ``65.6.3`` from ``65.5.1`` - by :user:`gaborbernat`. (`#2451 `_) +================== +- Do not use deprecated API from ``importlib.resources`` on Python 3.10 or later - by :user:`gaborbernat`. (`#2448 + `_) +- Upgrade embedded setuptools to ``65.6.3`` from ``65.5.1`` - by :user:`gaborbernat`. (`#2451 + `_) -v20.16.7 (2022-11-12) ---------------------- +*********************** + v20.16.7 (2022-11-12) +*********************** Bugfixes - 20.16.7 -~~~~~~~~~~~~~~~~~~ -- Use parent directory of python executable for pyvenv.cfg "home" value per PEP 405 - by :user:`vfazio`. (`#2440 `_) -- In POSIX virtual environments, try alternate binary names if ``sys._base_executable`` does not exist - by :user:`vfazio`. (`#2442 `_) -- Upgrade embedded wheel to ``0.38.4`` and pip to ``22.3.1`` from ``22.3`` and setuptools to ``65.5.1`` from - ``65.5.0`` - by :user:`gaborbernat`. (`#2443 `_) +================== +- Use parent directory of python executable for pyvenv.cfg "home" value per PEP 405 - by :user:`vfazio`. (`#2440 + `_) +- In POSIX virtual environments, try alternate binary names if ``sys._base_executable`` does not exist - by + :user:`vfazio`. (`#2442 `_) +- Upgrade embedded wheel to ``0.38.4`` and pip to ``22.3.1`` from ``22.3`` and setuptools to ``65.5.1`` from ``65.5.0`` + - by :user:`gaborbernat`. (`#2443 `_) -v20.16.6 (2022-10-25) ---------------------- +*********************** + v20.16.6 (2022-10-25) +*********************** Features - 20.16.6 -~~~~~~~~~~~~~~~~~~ +================== + - Drop unneeded shims for PyPy3 directory structure (`#2426 `_) Bugfixes - 20.16.6 -~~~~~~~~~~~~~~~~~~ -- Fix selected scheme on debian derivatives for python 3.10 when ``python3-distutils`` is not installed or the ``venv`` scheme is not available - by :user:`asottile`. (`#2350 `_) -- Allow the test suite to pass even with the original C shell (rather than ``tcsh``) - by :user:`kulikjak`. (`#2418 `_) -- Fix fallback handling of downloading wheels for bundled packages - by :user:`schaap`. (`#2429 `_) -- Upgrade embedded setuptools to ``65.5.0`` from ``65.3.0`` and pip to ``22.3`` from ``22.2.2`` - by :user:`gaborbernat`. (`#2434 `_) +================== +- Fix selected scheme on debian derivatives for python 3.10 when ``python3-distutils`` is not installed or the ``venv`` + scheme is not available - by :user:`asottile`. (`#2350 `_) +- Allow the test suite to pass even with the original C shell (rather than ``tcsh``) - by :user:`kulikjak`. (`#2418 + `_) +- Fix fallback handling of downloading wheels for bundled packages - by :user:`schaap`. (`#2429 + `_) +- Upgrade embedded setuptools to ``65.5.0`` from ``65.3.0`` and pip to ``22.3`` from ``22.2.2`` - by + :user:`gaborbernat`. (`#2434 `_) -v20.16.5 (2022-09-07) ---------------------- +*********************** + v20.16.5 (2022-09-07) +*********************** Bugfixes - 20.16.5 -~~~~~~~~~~~~~~~~~~ -- Do not turn echo off for subsequent commands in batch activators - (``activate.bat`` and ``deactivate.bat``) - by :user:`pawelszramowski`. (`#2411 `_) +================== +- Do not turn echo off for subsequent commands in batch activators (``activate.bat`` and ``deactivate.bat``) - by + :user:`pawelszramowski`. (`#2411 `_) -v20.16.4 (2022-08-29) ---------------------- +*********************** + v20.16.4 (2022-08-29) +*********************** Bugfixes - 20.16.4 -~~~~~~~~~~~~~~~~~~ -- Bump embed setuptools to ``65.3`` - by :user:`gaborbernat`. (`#2405 `_) +================== +- Bump embed setuptools to ``65.3`` - by :user:`gaborbernat`. (`#2405 + `_) -v20.16.3 (2022-08-04) ---------------------- +*********************** + v20.16.3 (2022-08-04) +*********************** Bugfixes - 20.16.3 -~~~~~~~~~~~~~~~~~~ -- Upgrade embedded pip to ``22.2.2`` from ``22.2.1`` and setuptools to ``63.4.1`` from ``63.2.0`` - by :user:`gaborbernat`. (`#2395 `_) +================== +- Upgrade embedded pip to ``22.2.2`` from ``22.2.1`` and setuptools to ``63.4.1`` from ``63.2.0`` - by + :user:`gaborbernat`. (`#2395 `_) -v20.16.2 (2022-07-27) ---------------------- +*********************** + v20.16.2 (2022-07-27) +*********************** Bugfixes - 20.16.2 -~~~~~~~~~~~~~~~~~~ -- Bump embedded pip from ``22.2`` to ``22.2.1`` - by :user:`gaborbernat`. (`#2391 `_) +================== +- Bump embedded pip from ``22.2`` to ``22.2.1`` - by :user:`gaborbernat`. (`#2391 + `_) -v20.16.1 (2022-07-26) ---------------------- +*********************** + v20.16.1 (2022-07-26) +*********************** Features - 20.16.1 -~~~~~~~~~~~~~~~~~~ -- Update Nushell activation scripts to version 0.67 - by :user:`kubouch`. (`#2386 `_) +================== +- Update Nushell activation scripts to version 0.67 - by :user:`kubouch`. (`#2386 + `_) -v20.16.0 (2022-07-25) ---------------------- +*********************** + v20.16.0 (2022-07-25) +*********************** Features - 20.16.0 -~~~~~~~~~~~~~~~~~~ -- Drop support for running under Python 2 (still can generate Python 2 environments) - by :user:`gaborbernat`. (`#2382 `_) -- Upgrade embedded pip to ``22.2`` from ``22.1.2`` and setuptools to ``63.2.0`` from ``62.6.0`` - - by :user:`gaborbernat`. (`#2383 `_) +================== +- Drop support for running under Python 2 (still can generate Python 2 environments) - by :user:`gaborbernat`. (`#2382 + `_) +- Upgrade embedded pip to ``22.2`` from ``22.1.2`` and setuptools to ``63.2.0`` from ``62.6.0`` - by + :user:`gaborbernat`. (`#2383 `_) -v20.15.1 (2022-06-28) ---------------------- +*********************** + v20.15.1 (2022-06-28) +*********************** Bugfixes - 20.15.1 -~~~~~~~~~~~~~~~~~~ -- Fix the incorrect operation when ``setuptools`` plugins output something into ``stdout``. (`#2335 `_) -- CPython3Windows creator ignores missing ``DLLs`` dir. (`#2368 `_) +================== +- Fix the incorrect operation when ``setuptools`` plugins output something into ``stdout``. (`#2335 + `_) +- CPython3Windows creator ignores missing ``DLLs`` dir. (`#2368 `_) -v20.15.0 (2022-06-25) ---------------------- +*********************** + v20.15.0 (2022-06-25) +*********************** Features - 20.15.0 -~~~~~~~~~~~~~~~~~~ -- Support for Windows embeddable Python package: includes ``python.zip`` in the creator sources - - by :user:`reksarka`. (`#1774 `_) +================== + +- Support for Windows embeddable Python package: includes ``python.zip`` in the creator sources - by + :user:`reksarka`. (`#1774 `_) Bugfixes - 20.15.0 -~~~~~~~~~~~~~~~~~~ -- Upgrade embedded setuptools to ``62.3.3`` from ``62.6.0`` and pip to ``22.1.2`` from ``22.0.4`` - - by :user:`gaborbernat`. (`#2348 `_) -- Use ``shlex.quote`` instead of deprecated ``pipes.quote`` in Python 3 - by :user:`frenzymadness`. (`#2351 `_) -- Fix Windows PyPy 3.6 - by :user:`reksarka`. (`#2363 `_) +================== +- Upgrade embedded setuptools to ``62.3.3`` from ``62.6.0`` and pip to ``22.1.2`` from ``22.0.4`` - by + :user:`gaborbernat`. (`#2348 `_) +- Use ``shlex.quote`` instead of deprecated ``pipes.quote`` in Python 3 - by :user:`frenzymadness`. (`#2351 + `_) +- Fix Windows PyPy 3.6 - by :user:`reksarka`. (`#2363 `_) -v20.14.1 (2022-04-11) ---------------------- +*********************** + v20.14.1 (2022-04-11) +*********************** Features - 20.14.1 -~~~~~~~~~~~~~~~~~~ -- Support for creating a virtual environment from a Python 2.7 framework on macOS 12 - by :user:`nickhutchinson`. (`#2284 `_) +================== + +- Support for creating a virtual environment from a Python 2.7 framework on macOS 12 - by :user:`nickhutchinson`. + (`#2284 `_) Bugfixes - 20.14.1 -~~~~~~~~~~~~~~~~~~ -- Upgrade embedded setuptools to ``62.1.0`` from ``61.0.0`` - by :user:`gaborbernat`. (`#2327 `_) +================== +- Upgrade embedded setuptools to ``62.1.0`` from ``61.0.0`` - by :user:`gaborbernat`. (`#2327 + `_) -v20.14.0 (2022-03-25) ---------------------- +*********************** + v20.14.0 (2022-03-25) +*********************** Features - 20.14.0 -~~~~~~~~~~~~~~~~~~ -- Support Nushell activation scripts with nu version ``0.60`` - by :user:`kubouch`. (`#2321 `_) +================== + +- Support Nushell activation scripts with nu version ``0.60`` - by :user:`kubouch`. (`#2321 + `_) Bugfixes - 20.14.0 -~~~~~~~~~~~~~~~~~~ -- Upgrade embedded setuptools to ``61.0.0`` from ``60.10.0`` - by :user:`gaborbernat`. (`#2322 `_) +================== + +- Upgrade embedded setuptools to ``61.0.0`` from ``60.10.0`` - by :user:`gaborbernat`. (`#2322 + `_) -v20.13.4 (2022-03-18) ---------------------- +*********************** + v20.13.4 (2022-03-18) +*********************** Bugfixes - 20.13.4 -~~~~~~~~~~~~~~~~~~ -- Improve performance of python startup inside created virtualenvs - by :user:`asottile`. (`#2317 `_) -- Upgrade embedded setuptools to ``60.10.0`` from ``60.9.3`` - by :user:`gaborbernat`. (`#2320 `_) +================== -v20.13.3 (2022-03-07) ---------------------- +- Improve performance of python startup inside created virtualenvs - by :user:`asottile`. (`#2317 + `_) +- Upgrade embedded setuptools to ``60.10.0`` from ``60.9.3`` - by :user:`gaborbernat`. (`#2320 + `_) + +*********************** + v20.13.3 (2022-03-07) +*********************** Bugfixes - 20.13.3 -~~~~~~~~~~~~~~~~~~ -- Avoid symlinking the contents of ``/usr`` into PyPy3.8+ virtualenvs - by :user:`stefanor`. (`#2310 `_) -- Bump embed pip from ``22.0.3`` to ``22.0.4`` - by :user:`gaborbernat`. (`#2311 `_) +================== +- Avoid symlinking the contents of ``/usr`` into PyPy3.8+ virtualenvs - by :user:`stefanor`. (`#2310 + `_) +- Bump embed pip from ``22.0.3`` to ``22.0.4`` - by :user:`gaborbernat`. (`#2311 + `_) -v20.13.2 (2022-02-24) ---------------------- +*********************** + v20.13.2 (2022-02-24) +*********************** Bugfixes - 20.13.2 -~~~~~~~~~~~~~~~~~~ -- Upgrade embedded setuptools to ``60.9.3`` from ``60.6.0`` - by :user:`gaborbernat`. (`#2306 `_) +================== +- Upgrade embedded setuptools to ``60.9.3`` from ``60.6.0`` - by :user:`gaborbernat`. (`#2306 + `_) -v20.13.1 (2022-02-05) ---------------------- +*********************** + v20.13.1 (2022-02-05) +*********************** Bugfixes - 20.13.1 -~~~~~~~~~~~~~~~~~~ -- fix "execv() arg 2 must contain only strings" error on M1 MacOS (`#2282 `_) -- Upgrade embedded setuptools to ``60.5.0`` from ``60.2.0`` - by :user:`asottile`. (`#2289 `_) -- Upgrade embedded pip to ``22.0.3`` and setuptools to ``60.6.0`` - by :user:`gaborbernat` and :user:`asottile`. (`#2294 `_) +================== +- fix "execv() arg 2 must contain only strings" error on M1 MacOS (`#2282 + `_) +- Upgrade embedded setuptools to ``60.5.0`` from ``60.2.0`` - by :user:`asottile`. (`#2289 + `_) +- Upgrade embedded pip to ``22.0.3`` and setuptools to ``60.6.0`` - by :user:`gaborbernat` and :user:`asottile`. (`#2294 + `_) -v20.13.0 (2022-01-02) ---------------------- +*********************** + v20.13.0 (2022-01-02) +*********************** Features - 20.13.0 -~~~~~~~~~~~~~~~~~~ -- Add downloaded wheel information in the relevant JSON embed file to - prevent additional downloads of the same wheel. - by :user:`mayeut`. (`#2268 `_) +================== + +- Add downloaded wheel information in the relevant JSON embed file to prevent additional downloads of the same wheel. - + by :user:`mayeut`. (`#2268 `_) Bugfixes - 20.13.0 -~~~~~~~~~~~~~~~~~~ -- Fix ``AttributeError: 'bool' object has no attribute 'error'`` when creating a - Python 2.x virtualenv on macOS - by ``moreati``. (`#2269 `_) -- Fix ``PermissionError: [Errno 1] Operation not permitted`` when creating a - Python 2.x virtualenv on macOS/arm64 - by ``moreati``. (`#2271 `_) +================== +- Fix ``AttributeError: 'bool' object has no attribute 'error'`` when creating a Python 2.x virtualenv on macOS - by + ``moreati``. (`#2269 `_) +- Fix ``PermissionError: [Errno 1] Operation not permitted`` when creating a Python 2.x virtualenv on macOS/arm64 - by + ``moreati``. (`#2271 `_) -v20.12.1 (2022-01-01) ---------------------- +*********************** + v20.12.1 (2022-01-01) +*********************** Bugfixes - 20.12.1 -~~~~~~~~~~~~~~~~~~ -- Try using previous updates of ``pip``, ``setuptools`` & ``wheel`` - when inside an update grace period rather than always falling back - to embedded wheels - by :user:`mayeut`. (`#2265 `_) -- New patch versions of ``pip``, ``setuptools`` & ``wheel`` are now - returned in the expected timeframe. - by :user:`mayeut`. (`#2266 `_) -- Manual upgrades of ``pip``, ``setuptools`` & ``wheel`` are - not discarded by a periodic update - by :user:`mayeut`. (`#2267 `_) +================== +- Try using previous updates of ``pip``, ``setuptools`` & ``wheel`` when inside an update grace period rather than + always falling back to embedded wheels - by :user:`mayeut`. (`#2265 + `_) +- New patch versions of ``pip``, ``setuptools`` & ``wheel`` are now returned in the expected timeframe. - by + :user:`mayeut`. (`#2266 `_) +- Manual upgrades of ``pip``, ``setuptools`` & ``wheel`` are not discarded by a periodic update - by :user:`mayeut`. + (`#2267 `_) -v20.12.0 (2021-12-31) ---------------------- +*********************** + v20.12.0 (2021-12-31) +*********************** Features - 20.12.0 -~~~~~~~~~~~~~~~~~~ -- Sign the python2 exe on Darwin arm64 - by :user:`tmspicer`. (`#2233 `_) +================== + +- Sign the python2 exe on Darwin arm64 - by :user:`tmspicer`. (`#2233 + `_) Bugfixes - 20.12.0 -~~~~~~~~~~~~~~~~~~ -- Fix ``--download`` option - by :user:`mayeut`. (`#2120 `_) -- Upgrade embedded setuptools to ``60.2.0`` from ``60.1.1`` - by :user:`gaborbernat`. (`#2263 `_) +================== +- Fix ``--download`` option - by :user:`mayeut`. (`#2120 `_) +- Upgrade embedded setuptools to ``60.2.0`` from ``60.1.1`` - by :user:`gaborbernat`. (`#2263 + `_) -v20.11.2 (2021-12-29) ---------------------- +*********************** + v20.11.2 (2021-12-29) +*********************** Bugfixes - 20.11.2 -~~~~~~~~~~~~~~~~~~ -- Fix installation of pinned versions of ``pip``, ``setuptools`` & ``wheel`` - by :user:`mayeut`. (`#2203 `_) +================== +- Fix installation of pinned versions of ``pip``, ``setuptools`` & ``wheel`` - by :user:`mayeut`. (`#2203 + `_) -v20.11.1 (2021-12-29) ---------------------- +*********************** + v20.11.1 (2021-12-29) +*********************** Bugfixes - 20.11.1 -~~~~~~~~~~~~~~~~~~ -- Bump embed setuptools to ``60.1.1`` from ``60.1.0`` - by :user:`gaborbernat`. (`#2258 `_) +================== +- Bump embed setuptools to ``60.1.1`` from ``60.1.0`` - by :user:`gaborbernat`. (`#2258 + `_) -v20.11.0 (2021-12-28) ---------------------- +*********************** + v20.11.0 (2021-12-28) +*********************** Features - 20.11.0 -~~~~~~~~~~~~~~~~~~ -- Avoid deprecation warning from py-filelock argument - by :user:`ofek`. (`#2237 `_) -- Upgrade embedded setuptools to ``61.1.0`` from ``58.3.0`` - by :user:`gaborbernat`. (`#2240 `_) -- Drop the runtime dependency of ``backports.entry-points-selectable`` - by :user:`hroncok`. (`#2246 `_) -- Fish: PATH variables should not be quoted when being set - by :user:`d3dave`. (`#2248 `_) +================== +- Avoid deprecation warning from py-filelock argument - by :user:`ofek`. (`#2237 + `_) +- Upgrade embedded setuptools to ``61.1.0`` from ``58.3.0`` - by :user:`gaborbernat`. (`#2240 + `_) +- Drop the runtime dependency of ``backports.entry-points-selectable`` - by :user:`hroncok`. (`#2246 + `_) +- Fish: PATH variables should not be quoted when being set - by :user:`d3dave`. (`#2248 + `_) -v20.10.0 (2021-11-01) ---------------------- +*********************** + v20.10.0 (2021-11-01) +*********************** Features - 20.10.0 -~~~~~~~~~~~~~~~~~~ +================== + - If a ``"venv"`` install scheme exists in ``sysconfig``, virtualenv now uses it to create new virtual environments. - This allows Python distributors, such as Fedora, to patch/replace the default install scheme without affecting - the paths in new virtual environments. - A similar technique `was proposed to Python, for the venv module `_ - by ``hroncok`` (`#2208 `_) -- The activated virtualenv prompt is now always wrapped in parentheses. This - affects venvs created with the ``--prompt`` attribute, and matches virtualenv's - behavior on par with venv. (`#2224 `_) + This allows Python distributors, such as Fedora, to patch/replace the default install scheme without affecting the + paths in new virtual environments. A similar technique `was proposed to Python, for the venv module + `_ - by ``hroncok`` (`#2208 `_) +- The activated virtualenv prompt is now always wrapped in parentheses. This affects venvs created with the ``--prompt`` + attribute, and matches virtualenv's behavior on par with venv. (`#2224 + `_) Bugfixes - 20.10.0 -~~~~~~~~~~~~~~~~~~ -- Fix broken prompt set up by activate.bat - by :user:`SiggyBar`. (`#2225 `_) +================== +- Fix broken prompt set up by activate.bat - by :user:`SiggyBar`. (`#2225 + `_) -v20.9.0 (2021-10-23) --------------------- +********************** + v20.9.0 (2021-10-23) +********************** Features - 20.9.0 -~~~~~~~~~~~~~~~~~ -- Special-case ``--prompt .`` to the name of the current directory - by :user:`rkm`. (`#2220 `_) +================= + +- Special-case ``--prompt .`` to the name of the current directory - by :user:`rkm`. (`#2220 + `_) - Add libffi-8.dll to pypy windows `#2218 `_ - by :user:`mattip` Bugfixes - 20.9.0 -~~~~~~~~~~~~~~~~~ -- Fixed path collision that could lead to a PermissionError or writing to system - directories when using PyPy3.8 - by :user:`mgorny`. (`#2182 `_) +================= + +- Fixed path collision that could lead to a PermissionError or writing to system directories when using PyPy3.8 - by + :user:`mgorny`. (`#2182 `_) - Upgrade embedded setuptools to ``58.3.0`` from ``58.1.0`` and pip to ``21.3.1`` from ``21.2.4`` - by :user:`gaborbernat`. (`#2205 `_) -- Remove stray closing parenthesis in activate.bat - by :user:`SiggyBar`. (`#2221 `_) +- Remove stray closing parenthesis in activate.bat - by :user:`SiggyBar`. (`#2221 + `_) - -v20.8.1 (2021-09-24) --------------------- +********************** + v20.8.1 (2021-09-24) +********************** Bugfixes - 20.8.1 -~~~~~~~~~~~~~~~~~ -- Fixed a bug where while creating a venv on top of an existing one, without cleaning, when seeded - wheel version mismatch occurred, multiple ``.dist-info`` directories may be present, confounding entrypoint - discovery - by :user:`arcivanov` (`#2185 `_) -- Bump embed setuptools from ``58.0.4`` to ``58.1.0`` - by :user:`gaborbernat`. (`#2195 `_) +================= + +- Fixed a bug where while creating a venv on top of an existing one, without cleaning, when seeded wheel version + mismatch occurred, multiple ``.dist-info`` directories may be present, confounding entrypoint discovery - by + :user:`arcivanov` (`#2185 `_) +- Bump embed setuptools from ``58.0.4`` to ``58.1.0`` - by :user:`gaborbernat`. (`#2195 + `_) Misc - 20.8.1 -~~~~~~~~~~~~~ -- `#2189 `_ +============= +- `#2189 `_ -v20.8.0 (2021-09-16) --------------------- +********************** + v20.8.0 (2021-09-16) +********************** -* upgrade embedded setuptools to ``58.0.4`` from ``57.4.0`` and pip to ``21.2.4`` from ``21.2.3`` -* Add nushell activation script +- upgrade embedded setuptools to ``58.0.4`` from ``57.4.0`` and pip to ``21.2.4`` from ``21.2.3`` +- Add nushell activation script -v20.7.2 (2021-08-10) --------------------- +********************** + v20.7.2 (2021-08-10) +********************** Bugfixes - 20.7.2 -~~~~~~~~~~~~~~~~~ -- Upgrade embedded pip to ``21.2.3`` from ``21.2.2`` and wheel to ``0.37.0`` from ``0.36.2`` - by :user:`gaborbernat`. (`#2168 `_) +================= +- Upgrade embedded pip to ``21.2.3`` from ``21.2.2`` and wheel to ``0.37.0`` from ``0.36.2`` - by :user:`gaborbernat`. + (`#2168 `_) -v20.7.1 (2021-08-09) --------------------- +********************** + v20.7.1 (2021-08-09) +********************** Bugfixes - 20.7.1 -~~~~~~~~~~~~~~~~~ -- Fix unpacking dictionary items in PythonInfo.install_path (`#2165 `_) +================= +- Fix unpacking dictionary items in PythonInfo.install_path (`#2165 `_) -v20.7.0 (2021-07-31) --------------------- +********************** + v20.7.0 (2021-07-31) +********************** Bugfixes - 20.7.0 -~~~~~~~~~~~~~~~~~ -- upgrade embedded pip to ``21.2.2`` from ``21.1.3`` and setuptools to ``57.4.0`` from ``57.1.0`` - by :user:`gaborbernat` (`#2159 `_) +================= + +- upgrade embedded pip to ``21.2.2`` from ``21.1.3`` and setuptools to ``57.4.0`` from ``57.1.0`` - by + :user:`gaborbernat` (`#2159 `_) Deprecations and Removals - 20.7.0 -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -- Removed ``xonsh`` activator due to this breaking fairly often the CI and lack of support from those packages - maintainers, upstream is encouraged to continue supporting the project as a - `plugin `_ - by :user:`gaborbernat`. (`#2160 `_) +================================== +- Removed ``xonsh`` activator due to this breaking fairly often the CI and lack of support from those packages + maintainers, upstream is encouraged to continue supporting the project as a `plugin + `_ - by :user:`gaborbernat`. (`#2160 + `_) -v20.6.0 (2021-07-14) --------------------- +********************** + v20.6.0 (2021-07-14) +********************** Features - 20.6.0 -~~~~~~~~~~~~~~~~~ -- Support Python interpreters without ``distutils`` (fallback to ``syconfig`` in these cases) - by :user:`gaborbernat`. (`#1910 `_) +================= +- Support Python interpreters without ``distutils`` (fallback to ``syconfig`` in these cases) - by :user:`gaborbernat`. + (`#1910 `_) -v20.5.0 (2021-07-13) --------------------- +********************** + v20.5.0 (2021-07-13) +********************** Features - 20.5.0 -~~~~~~~~~~~~~~~~~ -- Plugins now use 'selectable' entry points - by :user:`jaraco`. (`#2093 `_) +================= + +- Plugins now use 'selectable' entry points - by :user:`jaraco`. (`#2093 + `_) - add libffi-7.dll to the hard-coded list of dlls for PyPy (`#2141 `_) -- Use the better maintained ``platformdirs`` instead of ``appdirs`` - by :user:`gaborbernat`. (`#2142 `_) +- Use the better maintained ``platformdirs`` instead of ``appdirs`` - by :user:`gaborbernat`. (`#2142 + `_) Bugfixes - 20.5.0 -~~~~~~~~~~~~~~~~~ -- Bump pip the embedded pip ``21.1.3`` and setuptools to ``57.1.0`` - by :user:`gaborbernat`. (`#2135 `_) +================= + +- Bump pip the embedded pip ``21.1.3`` and setuptools to ``57.1.0`` - by :user:`gaborbernat`. (`#2135 + `_) Deprecations and Removals - 20.5.0 -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -- Drop python ``3.4`` support as it has been over 2 years since EOL - by :user:`gaborbernat`. (`#2141 `_) +================================== +- Drop python ``3.4`` support as it has been over 2 years since EOL - by :user:`gaborbernat`. (`#2141 + `_) -v20.4.7 (2021-05-24) --------------------- +********************** + v20.4.7 (2021-05-24) +********************** Bugfixes - 20.4.7 -~~~~~~~~~~~~~~~~~ -- Upgrade embedded pip to ``21.1.2`` and setuptools to ``57.0.0`` - by :user:`gaborbernat`. (`#2123 `_) +================= +- Upgrade embedded pip to ``21.1.2`` and setuptools to ``57.0.0`` - by :user:`gaborbernat`. (`#2123 + `_) -v20.4.6 (2021-05-05) --------------------- +********************** + v20.4.6 (2021-05-05) +********************** Bugfixes - 20.4.6 -~~~~~~~~~~~~~~~~~ -- Fix ``site.getsitepackages()`` broken on python2 on debian - by :user:`freundTech`. (`#2105 `_) +================= +- Fix ``site.getsitepackages()`` broken on python2 on debian - by :user:`freundTech`. (`#2105 + `_) -v20.4.5 (2021-05-05) --------------------- +********************** + v20.4.5 (2021-05-05) +********************** Bugfixes - 20.4.5 -~~~~~~~~~~~~~~~~~ -- Bump pip to ``21.1.1`` from ``21.0.1`` - by :user:`gaborbernat`. (`#2104 `_) -- Fix ``site.getsitepackages()`` ignoring ``--system-site-packages`` on python2 - by :user:`freundTech`. (`#2106 `_) +================= +- Bump pip to ``21.1.1`` from ``21.0.1`` - by :user:`gaborbernat`. (`#2104 + `_) +- Fix ``site.getsitepackages()`` ignoring ``--system-site-packages`` on python2 - by :user:`freundTech`. (`#2106 + `_) -v20.4.4 (2021-04-20) --------------------- +********************** + v20.4.4 (2021-04-20) +********************** Bugfixes - 20.4.4 -~~~~~~~~~~~~~~~~~ -- Built in discovery class is always preferred over plugin supplied classes. (`#2087 `_) -- Upgrade embedded setuptools to ``56.0.0`` by :user:`gaborbernat`. (`#2094 `_) +================= +- Built in discovery class is always preferred over plugin supplied classes. (`#2087 + `_) +- Upgrade embedded setuptools to ``56.0.0`` by :user:`gaborbernat`. (`#2094 + `_) -v20.4.3 (2021-03-16) --------------------- +********************** + v20.4.3 (2021-03-16) +********************** Bugfixes - 20.4.3 -~~~~~~~~~~~~~~~~~ -- Bump embedded setuptools from ``52.0.0`` to ``54.1.2`` - by :user:`gaborbernat` (`#2069 `_) -- Fix PyPy3 stdlib on Windows is incorrect - by :user:`gaborbernat`. (`#2071 `_) +================= +- Bump embedded setuptools from ``52.0.0`` to ``54.1.2`` - by :user:`gaborbernat` (`#2069 + `_) +- Fix PyPy3 stdlib on Windows is incorrect - by :user:`gaborbernat`. (`#2071 + `_) -v20.4.2 (2021-02-01) --------------------- +********************** + v20.4.2 (2021-02-01) +********************** Bugfixes - 20.4.2 -~~~~~~~~~~~~~~~~~ -- Running virtualenv ``--upgrade-embed-wheels`` crashes - by :user:`gaborbernat`. (`#2058 `_) +================= +- Running virtualenv ``--upgrade-embed-wheels`` crashes - by :user:`gaborbernat`. (`#2058 + `_) -v20.4.1 (2021-01-31) --------------------- +********************** + v20.4.1 (2021-01-31) +********************** Bugfixes - 20.4.1 -~~~~~~~~~~~~~~~~~ -- Bump embedded pip and setuptools packages to latest upstream supported (``21.0.1`` and ``52.0.0``) - by :user:`gaborbernat`. (`#2060 `_) +================= +- Bump embedded pip and setuptools packages to latest upstream supported (``21.0.1`` and ``52.0.0``) - by + :user:`gaborbernat`. (`#2060 `_) -v20.4.0 (2021-01-19) --------------------- +********************** + v20.4.0 (2021-01-19) +********************** Features - 20.4.0 -~~~~~~~~~~~~~~~~~ +================= + - On the programmatic API allow passing in the environment variable dictionary to use, defaults to ``os.environ`` if not specified - by :user:`gaborbernat`. (`#2054 `_) Bugfixes - 20.4.0 -~~~~~~~~~~~~~~~~~ -- Upgrade embedded setuptools to ``51.3.3`` from ``51.1.2`` - by :user:`gaborbernat`. (`#2055 `_) +================= +- Upgrade embedded setuptools to ``51.3.3`` from ``51.1.2`` - by :user:`gaborbernat`. (`#2055 + `_) -v20.3.1 (2021-01-13) --------------------- +********************** + v20.3.1 (2021-01-13) +********************** Bugfixes - 20.3.1 -~~~~~~~~~~~~~~~~~ -- Bump embed pip to ``20.3.3``, setuptools to ``51.1.1`` and wheel to ``0.36.2`` - by :user:`gaborbernat`. (`#2036 `_) -- Allow unfunctioning of pydoc to fail freely so that virtualenvs can be - activated under Zsh with set -e (since otherwise ``unset -f`` and - ``unfunction`` exit with 1 if the function does not exist in Zsh) - by - :user:`d125q`. (`#2049 `_) -- Drop cached python information if the system executable is no longer present (for example when the executable is a - shim and the mapped executable is replaced - such is the case with pyenv) - by :user:`gaborbernat`. (`#2050 `_) +================= +- Bump embed pip to ``20.3.3``, setuptools to ``51.1.1`` and wheel to ``0.36.2`` - by :user:`gaborbernat`. (`#2036 + `_) +- Allow unfunctioning of pydoc to fail freely so that virtualenvs can be activated under Zsh with set -e (since + otherwise ``unset -f`` and ``unfunction`` exit with 1 if the function does not exist in Zsh) - by :user:`d125q`. + (`#2049 `_) +- Drop cached python information if the system executable is no longer present (for example when the executable is a + shim and the mapped executable is replaced - such is the case with pyenv) - by :user:`gaborbernat`. (`#2050 + `_) -v20.3.0 (2021-01-10) --------------------- +********************** + v20.3.0 (2021-01-10) +********************** Features - 20.3.0 -~~~~~~~~~~~~~~~~~ +================= + - The builtin discovery takes now a ``--try-first-with`` argument and is first attempted as valid interpreters. One can - use this to force discovery of a given python executable when the discovery order/mechanism raises errors - - by :user:`gaborbernat`. (`#2046 `_) + use this to force discovery of a given python executable when the discovery order/mechanism raises errors - by + :user:`gaborbernat`. (`#2046 `_) Bugfixes - 20.3.0 -~~~~~~~~~~~~~~~~~ -- On Windows python ``3.7+`` distributions where the exe shim is missing fallback to the old ways - by :user:`gaborbernat`. (`#1986 `_) -- When discovering interpreters on Windows, via the PEP-514, prefer ``PythonCore`` releases over other ones. virtualenv - is used via pip mostly by this distribution, so prefer it over other such as conda - by :user:`gaborbernat`. (`#2046 `_) +================= +- On Windows python ``3.7+`` distributions where the exe shim is missing fallback to the old ways - by + :user:`gaborbernat`. (`#1986 `_) +- When discovering interpreters on Windows, via the PEP-514, prefer ``PythonCore`` releases over other ones. virtualenv + is used via pip mostly by this distribution, so prefer it over other such as conda - by :user:`gaborbernat`. (`#2046 + `_) -v20.2.2 (2020-12-07) --------------------- +********************** + v20.2.2 (2020-12-07) +********************** Bugfixes - 20.2.2 -~~~~~~~~~~~~~~~~~ -- Bump pip to ``20.3.1``, setuptools to ``51.0.0`` and wheel to ``0.36.1`` - by :user:`gaborbernat`. (`#2029 `_) +================= +- Bump pip to ``20.3.1``, setuptools to ``51.0.0`` and wheel to ``0.36.1`` - by :user:`gaborbernat`. (`#2029 + `_) -v20.2.1 (2020-11-23) --------------------- +********************** + v20.2.1 (2020-11-23) +********************** No significant changes. - -v20.2.0 (2020-11-21) --------------------- +********************** + v20.2.0 (2020-11-21) +********************** Features - 20.2.0 -~~~~~~~~~~~~~~~~~ -- Optionally skip VCS ignore directive for entire virtualenv directory, using option :option:`no-vcs-ignore`, by default ``False``. (`#2003 `_) -- Add ``--read-only-app-data`` option to allow for creation based on an existing - app data cache which is non-writable. This may be useful (for example) to - produce a docker image where the app-data is pre-populated. +================= + +- Optionally skip VCS ignore directive for entire virtualenv directory, using option :option:`no-vcs-ignore`, by default + ``False``. (`#2003 `_) +- Add ``--read-only-app-data`` option to allow for creation based on an existing app data cache which is non-writable. + This may be useful (for example) to produce a docker image where the app-data is pre-populated. .. code-block:: dockerfile @@ -1073,114 +2176,147 @@ Features - 20.2.0 Patch by :user:`asottile`. (`#2009 `_) Bugfixes - 20.2.0 -~~~~~~~~~~~~~~~~~ -- Fix processing of the ``VIRTUALENV_PYTHON`` environment variable and make it - multi-value as well (separated by comma) - by :user:`pneff`. (`#1998 `_) +================= +- Fix processing of the ``VIRTUALENV_PYTHON`` environment variable and make it multi-value as well (separated by comma) + - by :user:`pneff`. (`#1998 `_) -v20.1.0 (2020-10-25) --------------------- +********************** + v20.1.0 (2020-10-25) +********************** Features - 20.1.0 -~~~~~~~~~~~~~~~~~ +================= + - The python specification can now take one or more values, first found is used to create the virtual environment - by :user:`gaborbernat`. (`#1995 `_) - -v20.0.35 (2020-10-15) ---------------------- +*********************** + v20.0.35 (2020-10-15) +*********************** Bugfixes - 20.0.35 -~~~~~~~~~~~~~~~~~~ -- Bump embedded setuptools from ``50.3.0`` to ``50.3.1`` - by :user:`gaborbernat`. (`#1982 `_) -- After importing virtualenv passing cwd to a subprocess calls breaks with ``invalid directory`` - by :user:`gaborbernat`. (`#1983 `_) +================== +- Bump embedded setuptools from ``50.3.0`` to ``50.3.1`` - by :user:`gaborbernat`. (`#1982 + `_) +- After importing virtualenv passing cwd to a subprocess calls breaks with ``invalid directory`` - by + :user:`gaborbernat`. (`#1983 `_) -v20.0.34 (2020-10-12) ---------------------- +*********************** + v20.0.34 (2020-10-12) +*********************** Bugfixes - 20.0.34 -~~~~~~~~~~~~~~~~~~ -- Align with venv module when creating virtual environments with builtin creator on Windows 3.7 and later - - by :user:`gaborbernat`. (`#1782 `_) -- Handle Cygwin path conversion in the activation script - by :user:`davidcoghlan`. (`#1969 `_) +================== +- Align with venv module when creating virtual environments with builtin creator on Windows 3.7 and later - by + :user:`gaborbernat`. (`#1782 `_) +- Handle Cygwin path conversion in the activation script - by :user:`davidcoghlan`. (`#1969 + `_) -v20.0.33 (2020-10-04) ---------------------- +*********************** + v20.0.33 (2020-10-04) +*********************** Bugfixes - 20.0.33 -~~~~~~~~~~~~~~~~~~ -- Fix ``None`` type error in cygwin if POSIX path in dest - by :user:`danyeaw`. (`#1962 `_) -- Fix Python 3.4 incompatibilities (added back to the CI) - by :user:`gaborbernat`. (`#1963 `_) +================== +- Fix ``None`` type error in cygwin if POSIX path in dest - by :user:`danyeaw`. (`#1962 + `_) +- Fix Python 3.4 incompatibilities (added back to the CI) - by :user:`gaborbernat`. (`#1963 + `_) -v20.0.32 (2020-10-01) ---------------------- +*********************** + v20.0.32 (2020-10-01) +*********************** Bugfixes - 20.0.32 -~~~~~~~~~~~~~~~~~~ -- For activation scripts always use UNIX line endings (unless it's BATCH shell related) - by :user:`saytosid`. (`#1818 `_) -- Upgrade embedded pip to ``20.2.1`` and setuptools to ``49.4.0`` - by :user:`gaborbernat`. (`#1918 `_) -- Avoid spawning new windows when doing seed package upgrades in the background on Windows - by :user:`gaborbernat`. (`#1928 `_) -- Fix a bug that reading and writing on the same file may cause race on multiple processes. (`#1938 `_) -- Upgrade embedded setuptools to ``50.2.0`` and pip to ``20.2.3`` - by :user:`gaborbernat`. (`#1939 `_) -- Provide correct path for bash activator in cygwin or msys2 - by :user:`danyeaw`. (`#1940 `_) -- Relax importlib requirement to allow version<3 - by :user:`usamasadiq` (`#1953 `_) -- pth files were not processed on CPython2 if $PYTHONPATH was pointing to site-packages/ - by :user:`navytux`. (`#1959 `_) (`#1960 `_) - - -v20.0.31 (2020-08-17) ---------------------- +================== + +- For activation scripts always use UNIX line endings (unless it's BATCH shell related) - by :user:`saytosid`. (`#1818 + `_) +- Upgrade embedded pip to ``20.2.1`` and setuptools to ``49.4.0`` - by :user:`gaborbernat`. (`#1918 + `_) +- Avoid spawning new windows when doing seed package upgrades in the background on Windows - by :user:`gaborbernat`. + (`#1928 `_) +- Fix a bug that reading and writing on the same file may cause race on multiple processes. (`#1938 + `_) +- Upgrade embedded setuptools to ``50.2.0`` and pip to ``20.2.3`` - by :user:`gaborbernat`. (`#1939 + `_) +- Provide correct path for bash activator in cygwin or msys2 - by :user:`danyeaw`. (`#1940 + `_) +- Relax importlib requirement to allow version<3 - by :user:`usamasadiq` (`#1953 + `_) +- pth files were not processed on CPython2 if $PYTHONPATH was pointing to site-packages/ - by :user:`navytux`. (`#1959 + `_) (`#1960 `_) + +*********************** + v20.0.31 (2020-08-17) +*********************** Bugfixes - 20.0.31 -~~~~~~~~~~~~~~~~~~ -- Upgrade embedded pip to ``20.2.1``, setuptools to ``49.6.0`` and wheel to ``0.35.1`` - by :user:`gaborbernat`. (`#1918 `_) +================== +- Upgrade embedded pip to ``20.2.1``, setuptools to ``49.6.0`` and wheel to ``0.35.1`` - by :user:`gaborbernat`. (`#1918 + `_) -v20.0.30 (2020-08-04) ---------------------- +*********************** + v20.0.30 (2020-08-04) +*********************** Bugfixes - 20.0.30 -~~~~~~~~~~~~~~~~~~ -- Upgrade pip to ``20.2.1`` and setuptools to ``49.2.1`` - by :user:`gaborbernat`. (`#1915 `_) +================== +- Upgrade pip to ``20.2.1`` and setuptools to ``49.2.1`` - by :user:`gaborbernat`. (`#1915 + `_) -v20.0.29 (2020-07-31) ---------------------- +*********************** + v20.0.29 (2020-07-31) +*********************** Bugfixes - 20.0.29 -~~~~~~~~~~~~~~~~~~ -- Upgrade embedded pip from version ``20.1.2`` to ``20.2`` - by :user:`gaborbernat`. (`#1909 `_) +================== +- Upgrade embedded pip from version ``20.1.2`` to ``20.2`` - by :user:`gaborbernat`. (`#1909 + `_) -v20.0.28 (2020-07-24) ---------------------- +*********************** + v20.0.28 (2020-07-24) +*********************** Bugfixes - 20.0.28 -~~~~~~~~~~~~~~~~~~ -- Fix test suite failing if run from system Python - by :user:`gaborbernat`. (`#1882 `_) +================== + +- Fix test suite failing if run from system Python - by :user:`gaborbernat`. (`#1882 + `_) - Provide ``setup_logging`` flag to python API so that users can bypass logging handling if their application already performs this - by :user:`gaborbernat`. (`#1896 `_) - Use ``\n`` instead if ``\r\n`` as line separator for report (because Python already performs this transformation - automatically upon write to the logging pipe) - by :user:`gaborbernat`. (`#1905 `_) - + automatically upon write to the logging pipe) - by :user:`gaborbernat`. (`#1905 + `_) -v20.0.27 (2020-07-15) ---------------------- +*********************** + v20.0.27 (2020-07-15) +*********************** Bugfixes - 20.0.27 -~~~~~~~~~~~~~~~~~~ -- No longer preimport threading to fix support for `gpython `_ and `gevent `_ - by :user:`navytux`. (`#1897 `_) -- Upgrade setuptools from ``49.2.0`` on ``Python 3.5+`` - by :user:`gaborbernat`. (`#1898 `_) +================== +- No longer preimport threading to fix support for `gpython `_ and `gevent + `_ - by :user:`navytux`. (`#1897 `_) +- Upgrade setuptools from ``49.2.0`` on ``Python 3.5+`` - by :user:`gaborbernat`. (`#1898 + `_) -v20.0.26 (2020-07-07) ---------------------- +*********************** + v20.0.26 (2020-07-07) +*********************** Bugfixes - 20.0.26 -~~~~~~~~~~~~~~~~~~ -- Bump dependency ``distutils >= 0.3.1`` - by :user:`gaborbernat`. (`#1880 `_) +================== + +- Bump dependency ``distutils >= 0.3.1`` - by :user:`gaborbernat`. (`#1880 + `_) - Improve periodic update handling: - better logging output while running and enable logging on background process call ( @@ -1189,36 +2325,45 @@ Bugfixes - 20.0.26 - stop downloading wheels once we reach the embedded version, by :user:`gaborbernat`. (`#1883 `_) -- Do not print error message if the application exists with ``SystemExit(0)`` - by :user:`gaborbernat`. (`#1885 `_) -- Upgrade embedded setuptools from ``47.3.1`` to ``49.1.0`` for Python ``3.5+`` - by :user:`gaborbernat`. (`#1887 `_) +- Do not print error message if the application exists with ``SystemExit(0)`` - by :user:`gaborbernat`. (`#1885 + `_) +- Upgrade embedded setuptools from ``47.3.1`` to ``49.1.0`` for Python ``3.5+`` - by :user:`gaborbernat`. (`#1887 + `_) -v20.0.25 (2020-06-23) ---------------------- +*********************** + v20.0.25 (2020-06-23) +*********************** Bugfixes - 20.0.25 -~~~~~~~~~~~~~~~~~~ -- Fix that when the ``app-data`` seeders image creation fails the exception is silently ignored. Avoid two virtual environment creations to step on each others toes by using a lock while creating the base images. By :user:`gaborbernat`. (`#1869 `_) +================== +- Fix that when the ``app-data`` seeders image creation fails the exception is silently ignored. Avoid two virtual + environment creations to step on each others toes by using a lock while creating the base images. By + :user:`gaborbernat`. (`#1869 `_) -v20.0.24 (2020-06-22) ---------------------- +*********************** + v20.0.24 (2020-06-22) +*********************** Features - 20.0.24 -~~~~~~~~~~~~~~~~~~ +================== + - Ensure that the seeded packages do not get too much out of date: - add a CLI flag that triggers upgrade of embedded wheels under :option:`upgrade-embed-wheels` - - periodically (once every 14 days) upgrade the embedded wheels in a background process, and use them if they have been - released for more than 28 days (can be disabled via :option:`no-periodic-update`) + - periodically (once every 14 days) upgrade the embedded wheels in a background process, and use them if they have + been released for more than 28 days (can be disabled via :option:`no-periodic-update`) More details under :ref:`wheels` - by :user:`gaborbernat`. (`#1821 `_) + - Upgrade embed wheel content: - ship wheels for Python ``3.9`` and ``3.10`` - upgrade setuptools for Python ``3.5+`` from ``47.1.1`` to ``47.3.1`` by :user:`gaborbernat`. (`#1841 `_) + - Display the installed seed package versions in the final summary output, for example: .. code-block:: console @@ -1231,249 +2376,317 @@ Features - 20.0.24 by :user:`gaborbernat`. (`#1864 `_) Bugfixes - 20.0.24 -~~~~~~~~~~~~~~~~~~ -- Do not generate/overwrite ``.gitignore`` if it already exists at destination path - by :user:`gaborbernat`. (`#1862 `_) -- Improve error message for no ``.dist-info`` inside the ``app-data`` copy seeder - by :user:`gaborbernat`. (`#1867 `_) +================== + +- Do not generate/overwrite ``.gitignore`` if it already exists at destination path - by :user:`gaborbernat`. (`#1862 + `_) +- Improve error message for no ``.dist-info`` inside the ``app-data`` copy seeder - by :user:`gaborbernat`. (`#1867 + `_) Improved Documentation - 20.0.24 -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -- How seeding mechanisms discover (and automatically keep it up to date) wheels at :ref:`wheels` - by :user:`gaborbernat`. (`#1821 `_) -- How distributions should handle shipping their own embedded wheels at :ref:`distribution_wheels` - by :user:`gaborbernat`. (`#1840 `_) +================================ +- How seeding mechanisms discover (and automatically keep it up to date) wheels at :ref:`wheels` - by + :user:`gaborbernat`. (`#1821 `_) +- How distributions should handle shipping their own embedded wheels at :ref:`distribution_wheels` - by + :user:`gaborbernat`. (`#1840 `_) -v20.0.23 (2020-06-12) ---------------------- +*********************** + v20.0.23 (2020-06-12) +*********************** Bugfixes - 20.0.23 -~~~~~~~~~~~~~~~~~~ -- Fix typo in ``setup.cfg`` - by :user:`RowdyHowell`. (`#1857 `_) +================== +- Fix typo in ``setup.cfg`` - by :user:`RowdyHowell`. (`#1857 `_) -v20.0.22 (2020-06-12) ---------------------- +*********************** + v20.0.22 (2020-06-12) +*********************** Bugfixes - 20.0.22 -~~~~~~~~~~~~~~~~~~ -- Relax ``importlib.resources`` requirement to also allow version 2 - by :user:`asottile`. (`#1846 `_) -- Upgrade embedded setuptools to ``44.1.1`` for python 2 and ``47.1.1`` for python3.5+ - by :user:`gaborbernat`. (`#1855 `_) +================== +- Relax ``importlib.resources`` requirement to also allow version 2 - by :user:`asottile`. (`#1846 + `_) +- Upgrade embedded setuptools to ``44.1.1`` for python 2 and ``47.1.1`` for python3.5+ - by :user:`gaborbernat`. (`#1855 + `_) -v20.0.21 (2020-05-20) ---------------------- +*********************** + v20.0.21 (2020-05-20) +*********************** Features - 20.0.21 -~~~~~~~~~~~~~~~~~~ +================== + - Generate ignore file for version control systems to avoid tracking virtual environments by default. Users should - remove these files if still want to track. For now we support only **git** by :user:`gaborbernat`. (`#1806 `_) + remove these files if still want to track. For now we support only **git** by :user:`gaborbernat`. (`#1806 + `_) Bugfixes - 20.0.21 -~~~~~~~~~~~~~~~~~~ +================== + - Fix virtualenv fails sometimes when run concurrently, ``--clear-app-data`` conflicts with :option:`clear` flag when abbreviation is turned on. To bypass this while allowing abbreviated flags on the command line we had to move it to :option:`reset-app-data` - by :user:`gaborbernat`. (`#1824 `_) -- Upgrade embedded ``setuptools`` to ``46.4.0`` from ``46.1.3`` on Python ``3.5+``, and ``pip`` from ``20.1`` to ``20.1.1`` - by :user:`gaborbernat`. (`#1827 `_) -- Seeder pip now correctly handles ``--extra-search-dir`` - by :user:`frenzymadness`. (`#1834 `_) - +- Upgrade embedded ``setuptools`` to ``46.4.0`` from ``46.1.3`` on Python ``3.5+``, and ``pip`` from ``20.1`` to + ``20.1.1`` - by :user:`gaborbernat`. (`#1827 `_) +- Seeder pip now correctly handles ``--extra-search-dir`` - by :user:`frenzymadness`. (`#1834 + `_) -v20.0.20 (2020-05-04) ---------------------- +*********************** + v20.0.20 (2020-05-04) +*********************** Bugfixes - 20.0.20 -~~~~~~~~~~~~~~~~~~ -- Fix download fails with python 3.4 - by :user:`gaborbernat`. (`#1809 `_) -- Fixes older CPython2 versions use ``_get_makefile_filename`` instead of ``get_makefile_filename`` on ``sysconfig`` - by :user:`ianw`. (`#1810 `_) -- Fix download is ``True`` by default - by :user:`gaborbernat`. (`#1813 `_) -- Fail ``app-data`` seed operation when wheel download fails and better error message - by :user:`gaborbernat`. (`#1814 `_) +================== +- Fix download fails with python 3.4 - by :user:`gaborbernat`. (`#1809 + `_) +- Fixes older CPython2 versions use ``_get_makefile_filename`` instead of ``get_makefile_filename`` on ``sysconfig`` - + by :user:`ianw`. (`#1810 `_) +- Fix download is ``True`` by default - by :user:`gaborbernat`. (`#1813 + `_) +- Fail ``app-data`` seed operation when wheel download fails and better error message - by :user:`gaborbernat`. (`#1814 + `_) -v20.0.19 (2020-05-03) ---------------------- +*********************** + v20.0.19 (2020-05-03) +*********************** Bugfixes - 20.0.19 -~~~~~~~~~~~~~~~~~~ -- Fix generating a Python 2 environment from Python 3 creates invalid python activator - by :user:`gaborbernat`. (`#1776 `_) -- Fix pinning seed packages via ``app-data`` seeder raised ``Invalid Requirement`` - by :user:`gaborbernat`. (`#1779 `_) -- Do not stop interpreter discovery if we fail to find the system interpreter for a executable during discovery - - by :user:`gaborbernat`. (`#1781 `_) -- On CPython2 POSIX platforms ensure ``syconfig.get_makefile_filename`` exists within the virtual environment (this is used by some c-extension based libraries - e.g. numpy - for building) - by :user:`gaborbernat`. (`#1783 `_) -- Better handling of options :option:`copies` and :option:`symlinks`. Introduce priority of where the option is set - to follow the order: CLI, env var, file, hardcoded. If both set at same level prefers copy over symlink. - by +================== + +- Fix generating a Python 2 environment from Python 3 creates invalid python activator - by :user:`gaborbernat`. (`#1776 + `_) +- Fix pinning seed packages via ``app-data`` seeder raised ``Invalid Requirement`` - by :user:`gaborbernat`. (`#1779 + `_) +- Do not stop interpreter discovery if we fail to find the system interpreter for a executable during discovery - by + :user:`gaborbernat`. (`#1781 `_) +- On CPython2 POSIX platforms ensure ``syconfig.get_makefile_filename`` exists within the virtual environment (this is + used by some c-extension based libraries - e.g. numpy - for building) - by :user:`gaborbernat`. (`#1783 + `_) +- Better handling of options :option:`copies` and :option:`symlinks`. Introduce priority of where the option is set to + follow the order: CLI, env var, file, hardcoded. If both set at same level prefers copy over symlink. - by :user:`gaborbernat`. (`#1784 `_) -- Upgrade pip for Python ``2.7`` and ``3.5+`` from ``20.0.2`` to ``20.1`` - by :user:`gaborbernat`. (`#1793 `_) -- Fix CPython is not discovered from Windows registry, and discover pythons from Windows registry in decreasing order - by version - by :user:`gaborbernat`. (`#1796 `_) +- Upgrade pip for Python ``2.7`` and ``3.5+`` from ``20.0.2`` to ``20.1`` - by :user:`gaborbernat`. (`#1793 + `_) +- Fix CPython is not discovered from Windows registry, and discover pythons from Windows registry in decreasing order by + version - by :user:`gaborbernat`. (`#1796 `_) - Fix symlink detection for creators - by :user:`asottile` (`#1803 `_) - -v20.0.18 (2020-04-16) ---------------------- +*********************** + v20.0.18 (2020-04-16) +*********************** Bugfixes - 20.0.18 -~~~~~~~~~~~~~~~~~~ +================== + - Importing setuptools before cli_run could cause our python information query to fail due to setuptools patching ``distutils.dist.Distribution`` - by :user:`gaborbernat`. (`#1771 `_) - -v20.0.17 (2020-04-09) ---------------------- +*********************** + v20.0.17 (2020-04-09) +*********************** Features - 20.0.17 -~~~~~~~~~~~~~~~~~~ -- Extend environment variables checked for configuration to also check aliases (e.g. setting either - ``VIRTUALENV_COPIES`` or ``VIRTUALENV_ALWAYS_COPY`` will work) - by :user:`gaborbernat`. (`#1763 `_) +================== +- Extend environment variables checked for configuration to also check aliases (e.g. setting either + ``VIRTUALENV_COPIES`` or ``VIRTUALENV_ALWAYS_COPY`` will work) - by :user:`gaborbernat`. (`#1763 + `_) -v20.0.16 (2020-04-04) ---------------------- +*********************** + v20.0.16 (2020-04-04) +*********************** Bugfixes - 20.0.16 -~~~~~~~~~~~~~~~~~~ -- Allow seed wheel files inside the :option:`extra-search-dir` folders that do not have ``Requires-Python`` - metadata specified, these are considered compatible with all python versions - by :user:`gaborbernat`. (`#1757 `_) +================== +- Allow seed wheel files inside the :option:`extra-search-dir` folders that do not have ``Requires-Python`` metadata + specified, these are considered compatible with all python versions - by :user:`gaborbernat`. (`#1757 + `_) -v20.0.15 (2020-03-27) ---------------------- +*********************** + v20.0.15 (2020-03-27) +*********************** Features - 20.0.15 -~~~~~~~~~~~~~~~~~~ -- Upgrade embedded setuptools to ``46.1.3`` from ``46.1.1`` - by :user:`gaborbernat`. (`#1752 `_) +================== +- Upgrade embedded setuptools to ``46.1.3`` from ``46.1.1`` - by :user:`gaborbernat`. (`#1752 + `_) -v20.0.14 (2020-03-25) ---------------------- +*********************** + v20.0.14 (2020-03-25) +*********************** Features - 20.0.14 -~~~~~~~~~~~~~~~~~~ +================== + - Remove ``__PYVENV_LAUNCHER__`` on macOs for Python ``3.7.(<8)`` and ``3.8.(<3)`` on interpreter startup via ``pth`` - file, this pulls in the `upstream patch `_ - by :user:`gaborbernat`. (`#1704 `_) -- Upgrade embedded setuptools for Python ``3.5+`` to ``46.1.1``, for Python ``2.7`` to ``44.1.0`` - by :user:`gaborbernat`. (`#1745 `_) + file, this pulls in the `upstream patch `_ - by :user:`gaborbernat`. + (`#1704 `_) +- Upgrade embedded setuptools for Python ``3.5+`` to ``46.1.1``, for Python ``2.7`` to ``44.1.0`` - by + :user:`gaborbernat`. (`#1745 `_) Bugfixes - 20.0.14 -~~~~~~~~~~~~~~~~~~ -- Fix discovery of interpreter by name from ``PATH`` that does not match a spec format - by :user:`gaborbernat`. (`#1746 `_) +================== +- Fix discovery of interpreter by name from ``PATH`` that does not match a spec format - by :user:`gaborbernat`. (`#1746 + `_) -v20.0.13 (2020-03-19) ---------------------- +*********************** + v20.0.13 (2020-03-19) +*********************** Bugfixes - 20.0.13 -~~~~~~~~~~~~~~~~~~ -- Do not fail when the pyc files is missing for the host Python 2 - by :user:`gaborbernat`. (`#1738 `_) -- Support broken Packaging pythons that put the include headers under distutils pattern rather than sysconfig one - - by :user:`gaborbernat`. (`#1739 `_) +================== +- Do not fail when the pyc files is missing for the host Python 2 - by :user:`gaborbernat`. (`#1738 + `_) +- Support broken Packaging pythons that put the include headers under distutils pattern rather than sysconfig one - by + :user:`gaborbernat`. (`#1739 `_) -v20.0.12 (2020-03-19) ---------------------- +*********************** + v20.0.12 (2020-03-19) +*********************** Bugfixes - 20.0.12 -~~~~~~~~~~~~~~~~~~ -- Fix relative path discovery of interpreters - by :user:`gaborbernat`. (`#1734 `_) +================== +- Fix relative path discovery of interpreters - by :user:`gaborbernat`. (`#1734 + `_) -v20.0.11 (2020-03-18) ---------------------- +*********************** + v20.0.11 (2020-03-18) +*********************** Features - 20.0.11 -~~~~~~~~~~~~~~~~~~ +================== + - Improve error message when the host python does not satisfy invariants needed to create virtual environments (now we - print which host files are incompatible/missing and for which creators when no supported creator can be matched, however - we found creators that can describe the given Python interpreter - will still print no supported creator for Jython, - however print exactly what host files do not allow creation of virtual environments in case of CPython/PyPy) - - by :user:`gaborbernat`. (`#1716 `_) + print which host files are incompatible/missing and for which creators when no supported creator can be matched, + however we found creators that can describe the given Python interpreter - will still print no supported creator for + Jython, however print exactly what host files do not allow creation of virtual environments in case of CPython/PyPy) - + by :user:`gaborbernat`. (`#1716 `_) Bugfixes - 20.0.11 -~~~~~~~~~~~~~~~~~~ -- Support Python 3 Framework distributed via XCode in macOs Catalina and before - by :user:`gaborbernat`. (`#1663 `_) -- Fix Windows Store Python support, do not allow creation via symlink as that's not going to work by design - - by :user:`gaborbernat`. (`#1709 `_) +================== + +- Support Python 3 Framework distributed via XCode in macOs Catalina and before - by :user:`gaborbernat`. (`#1663 + `_) +- Fix Windows Store Python support, do not allow creation via symlink as that's not going to work by design - by + :user:`gaborbernat`. (`#1709 `_) - Fix ``activate_this.py`` throws ``AttributeError`` on Windows when virtual environment was created via cross python mechanism - by :user:`gaborbernat`. (`#1710 `_) -- Fix ``--no-pip``, ``--no-setuptools``, ``--no-wheel`` not being respected - by :user:`gaborbernat`. (`#1712 `_) -- Allow missing ``.py`` files if a compiled ``.pyc`` version is available - by :user:`tucked`. (`#1714 `_) -- Do not fail if the distutils/setuptools patch happens on a C-extension loader (such as ``zipimporter`` on Python 3.7 or - earlier) - by :user:`gaborbernat`. (`#1715 `_) +- Fix ``--no-pip``, ``--no-setuptools``, ``--no-wheel`` not being respected - by :user:`gaborbernat`. (`#1712 + `_) +- Allow missing ``.py`` files if a compiled ``.pyc`` version is available - by :user:`tucked`. (`#1714 + `_) +- Do not fail if the distutils/setuptools patch happens on a C-extension loader (such as ``zipimporter`` on Python 3.7 + or earlier) - by :user:`gaborbernat`. (`#1715 `_) - Support Python 2 implementations that require the landmark files and ``site.py`` to be in platform standard library - instead of the standard library path of the virtual environment (notably some RHEL ones, such as the Docker - image ``amazonlinux:1``) - by :user:`gaborbernat`. (`#1719 `_) + instead of the standard library path of the virtual environment (notably some RHEL ones, such as the Docker image + ``amazonlinux:1``) - by :user:`gaborbernat`. (`#1719 `_) - Allow the test suite to pass even when called with the system Python - to help repackaging of the tool for Linux - distributions - by :user:`gaborbernat`. (`#1721 `_) -- Also generate ``pipx.y`` console script beside ``pip-x.y`` to be compatible with how pip installs itself - - by :user:`gaborbernat`. (`#1723 `_) -- Automatically create the application data folder if it does not exists - by :user:`gaborbernat`. (`#1728 `_) + distributions - by :user:`gaborbernat`. (`#1721 `_) +- Also generate ``pipx.y`` console script beside ``pip-x.y`` to be compatible with how pip installs itself - by + :user:`gaborbernat`. (`#1723 `_) +- Automatically create the application data folder if it does not exists - by :user:`gaborbernat`. (`#1728 + `_) Improved Documentation - 20.0.11 -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -- :ref:`supports ` details now explicitly what Python installations we support - - by :user:`gaborbernat`. (`#1714 `_) +================================ +- :ref:`supports ` details now explicitly what Python installations we support - by + :user:`gaborbernat`. (`#1714 `_) -v20.0.10 (2020-03-10) ---------------------- +*********************** + v20.0.10 (2020-03-10) +*********************** Bugfixes - 20.0.10 -~~~~~~~~~~~~~~~~~~ +================== + - Fix acquiring python information might be altered by distutils configuration files generating incorrect layout virtual environments - by :user:`gaborbernat`. (`#1663 `_) -- Upgrade embedded setuptools to ``46.0.0`` from ``45.3.0`` on Python ``3.5+`` - by :user:`gaborbernat`. (`#1702 `_) +- Upgrade embedded setuptools to ``46.0.0`` from ``45.3.0`` on Python ``3.5+`` - by :user:`gaborbernat`. (`#1702 + `_) Improved Documentation - 20.0.10 -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +================================ + - Document requirements (pip + index server) when installing via pip under the installation section - by :user:`gaborbernat`. (`#1618 `_) -- Document installing from non PEP-518 systems - :user:`gaborbernat`. (`#1619 `_) -- Document installing latest unreleased version from Github - :user:`gaborbernat`. (`#1620 `_) +- Document installing from non PEP-518 systems - :user:`gaborbernat`. (`#1619 + `_) +- Document installing latest unreleased version from Github - :user:`gaborbernat`. (`#1620 + `_) - -v20.0.9 (2020-03-08) --------------------- +********************** + v20.0.9 (2020-03-08) +********************** Bugfixes - 20.0.9 -~~~~~~~~~~~~~~~~~ -- ``pythonw.exe`` works as ``python.exe`` on Windows - by :user:`gaborbernat`. (`#1686 `_) -- Handle legacy loaders for virtualenv import hooks used to patch distutils configuration load - by :user:`gaborbernat`. (`#1690 `_) +================= + +- ``pythonw.exe`` works as ``python.exe`` on Windows - by :user:`gaborbernat`. (`#1686 + `_) +- Handle legacy loaders for virtualenv import hooks used to patch distutils configuration load - by :user:`gaborbernat`. + (`#1690 `_) - Support for python 2 platforms that store landmark files in ``platstdlib`` over ``stdlib`` (e.g. RHEL) - by :user:`gaborbernat`. (`#1694 `_) -- Upgrade embedded setuptools to ``45.3.0`` from ``45.2.0`` for Python ``3.5+`` - by :user:`gaborbernat`. (`#1699 `_) +- Upgrade embedded setuptools to ``45.3.0`` from ``45.2.0`` for Python ``3.5+`` - by :user:`gaborbernat`. (`#1699 + `_) - -v20.0.8 (2020-03-04) --------------------- +********************** + v20.0.8 (2020-03-04) +********************** Bugfixes - 20.0.8 -~~~~~~~~~~~~~~~~~ -- Having `distutils configuration `_ - files that set ``prefix`` and ``install_scripts`` cause installation of packages in the wrong location - - by :user:`gaborbernat`. (`#1663 `_) -- Fix ``PYTHONPATH`` being overridden on Python 2 — by :user:`jd`. (`#1673 `_) -- Fix list configuration value parsing from config file or environment variable - by :user:`gaborbernat`. (`#1674 `_) -- Fix Batch activation script shell prompt to display environment name by default - by :user:`spetafree`. (`#1679 `_) -- Fix startup on Python 2 is slower for virtualenv - this was due to setuptools calculating it's working set distribution - - by :user:`gaborbernat`. (`#1682 `_) +================= + +- Having `distutils configuration `_ files + that set ``prefix`` and ``install_scripts`` cause installation of packages in the wrong location - by + :user:`gaborbernat`. (`#1663 `_) +- Fix ``PYTHONPATH`` being overridden on Python 2 — by :user:`jd`. (`#1673 + `_) +- Fix list configuration value parsing from config file or environment variable - by :user:`gaborbernat`. (`#1674 + `_) +- Fix Batch activation script shell prompt to display environment name by default - by :user:`spetafree`. (`#1679 + `_) +- Fix startup on Python 2 is slower for virtualenv - this was due to setuptools calculating it's working set + distribution - by :user:`gaborbernat`. (`#1682 `_) - Fix entry points are not populated for editable installs on Python 2 due to setuptools working set being calculated before ``easy_install.pth`` runs - by :user:`gaborbernat`. (`#1684 `_) -- Fix ``attr:`` import fails for setuptools - by :user:`gaborbernat`. (`#1685 `_) - +- Fix ``attr:`` import fails for setuptools - by :user:`gaborbernat`. (`#1685 + `_) -v20.0.7 (2020-02-26) --------------------- +********************** + v20.0.7 (2020-02-26) +********************** Bugfixes - 20.0.7 -~~~~~~~~~~~~~~~~~ +================= + - Disable distutils fixup for python 3 until `pypa/pip #7778 `_ is fixed and released - by :user:`gaborbernat`. (`#1669 `_) - -v20.0.6 (2020-02-26) --------------------- +********************** + v20.0.6 (2020-02-26) +********************** Bugfixes - 20.0.6 -~~~~~~~~~~~~~~~~~ -- Fix global site package always being added with bundled macOs python framework builds - by :user:`gaborbernat`. (`#1561 `_) -- Fix generated scripts use host version info rather than target - by :user:`gaborbernat`. (`#1600 `_) -- Fix circular prefix reference with single elements (accept these as if they were system executables, print a info about - them referencing themselves) - by :user:`gaborbernat`. (`#1632 `_) +================= + +- Fix global site package always being added with bundled macOs python framework builds - by :user:`gaborbernat`. + (`#1561 `_) +- Fix generated scripts use host version info rather than target - by :user:`gaborbernat`. (`#1600 + `_) +- Fix circular prefix reference with single elements (accept these as if they were system executables, print a info + about them referencing themselves) - by :user:`gaborbernat`. (`#1632 + `_) - Handle the case when the application data folder is read-only: - the application data folder is now controllable via :option:`app-data`, @@ -1483,75 +2696,103 @@ Bugfixes - 20.0.6 - :option:`symlink-app-data` is always ``False`` when the application data is temporary by :user:`gaborbernat`. (`#1640 `_) -- Fix PyPy 2 builtin modules are imported from standard library, rather than from builtin - by :user:`gaborbernat`. (`#1652 `_) -- Fix creation of entry points when path contains spaces - by :user:`nsoranzo`. (`#1660 `_) -- Fix relative paths for the zipapp (for python ``3.7+``) - by :user:`gaborbernat`. (`#1666 `_) -v20.0.5 (2020-02-21) --------------------- +- Fix PyPy 2 builtin modules are imported from standard library, rather than from builtin - by :user:`gaborbernat`. + (`#1652 `_) +- Fix creation of entry points when path contains spaces - by :user:`nsoranzo`. (`#1660 + `_) +- Fix relative paths for the zipapp (for python ``3.7+``) - by :user:`gaborbernat`. (`#1666 + `_) + +********************** + v20.0.5 (2020-02-21) +********************** Features - 20.0.5 -~~~~~~~~~~~~~~~~~ -- Also create ``pythonX.X`` executables when creating pypy virtualenvs - by :user:`asottile` (`#1612 `_) -- Fail with better error message if trying to install source with unsupported ``setuptools``, allow ``setuptools-scm >= 2`` - and move to legacy ``setuptools-scm`` format to support better older platforms (``CentOS 7`` and such) - by :user:`gaborbernat`. (`#1621 `_) -- Report of the created virtual environment is now split across four short lines rather than one long - by :user:`gaborbernat` (`#1641 `_) +================= + +- Also create ``pythonX.X`` executables when creating pypy virtualenvs - by :user:`asottile` (`#1612 + `_) +- Fail with better error message if trying to install source with unsupported ``setuptools``, allow ``setuptools-scm >= + 2`` and move to legacy ``setuptools-scm`` format to support better older platforms (``CentOS 7`` and such) - by + :user:`gaborbernat`. (`#1621 `_) +- Report of the created virtual environment is now split across four short lines rather than one long - by + :user:`gaborbernat` (`#1641 `_) Bugfixes - 20.0.5 -~~~~~~~~~~~~~~~~~ -- Add macOs Python 2 Framework support (now we test it with the CI via brew) - by :user:`gaborbernat` (`#1561 `_) -- Fix losing of libpypy-c.so when the pypy executable is a symlink - by :user:`asottile` (`#1614 `_) -- Discover python interpreter in a case insensitive manner - by :user:`PrajwalM2212` (`#1624 `_) -- Fix cross interpreter support when the host python sets ``sys.base_executable`` based on ``__PYVENV_LAUNCHER__`` - - by :user:`cjolowicz` (`#1643 `_) +================= +- Add macOs Python 2 Framework support (now we test it with the CI via brew) - by :user:`gaborbernat` (`#1561 + `_) +- Fix losing of libpypy-c.so when the pypy executable is a symlink - by :user:`asottile` (`#1614 + `_) +- Discover python interpreter in a case insensitive manner - by :user:`PrajwalM2212` (`#1624 + `_) +- Fix cross interpreter support when the host python sets ``sys.base_executable`` based on ``__PYVENV_LAUNCHER__`` - by + :user:`cjolowicz` (`#1643 `_) -v20.0.4 (2020-02-14) --------------------- +********************** + v20.0.4 (2020-02-14) +********************** Features - 20.0.4 -~~~~~~~~~~~~~~~~~ -- When aliasing interpreters, use relative symlinks - by :user:`asottile`. (`#1596 `_) +================= + +- When aliasing interpreters, use relative symlinks - by :user:`asottile`. (`#1596 + `_) Bugfixes - 20.0.4 -~~~~~~~~~~~~~~~~~ -- Allow the use of ``/`` as pathname component separator on Windows - by ``vphilippon`` (`#1582 `_) -- Lower minimal version of six required to 1.9 - by ``ssbarnea`` (`#1606 `_) +================= +- Allow the use of ``/`` as pathname component separator on Windows - by ``vphilippon`` (`#1582 + `_) +- Lower minimal version of six required to 1.9 - by ``ssbarnea`` (`#1606 + `_) -v20.0.3 (2020-02-12) --------------------- +********************** + v20.0.3 (2020-02-12) +********************** Bugfixes - 20.0.3 -~~~~~~~~~~~~~~~~~ +================= + - On Python 2 with Apple Framework builds the global site package is no longer added when the - :option:`system-site-packages` is not specified - by :user:`gaborbernat`. (`#1561 `_) + :option:`system-site-packages` is not specified - by :user:`gaborbernat`. (`#1561 + `_) - Fix system python discovery mechanism when prefixes contain relative parts (e.g. ``..``) by resolving paths within the python information query - by :user:`gaborbernat`. (`#1583 `_) -- Expose a programmatic API as ``from virtualenv import cli_run`` - by :user:`gaborbernat`. (`#1585 `_) +- Expose a programmatic API as ``from virtualenv import cli_run`` - by :user:`gaborbernat`. (`#1585 + `_) - Fix ``app-data`` :option:`seeder` injects a extra ``.dist-info.virtualenv`` path that breaks ``importlib.metadata``, - now we inject an extra ``.virtualenv`` - by :user:`gaborbernat`. (`#1589 `_) + now we inject an extra ``.virtualenv`` - by :user:`gaborbernat`. (`#1589 + `_) Improved Documentation - 20.0.3 -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -- Document a programmatic API as ``from virtualenv import cli_run`` under :ref:`programmatic_api` - - by :user:`gaborbernat`. (`#1585 `_) +=============================== +- Document a programmatic API as ``from virtualenv import cli_run`` under :ref:`programmatic_api` - by + :user:`gaborbernat`. (`#1585 `_) -v20.0.2 (2020-02-11) --------------------- +********************** + v20.0.2 (2020-02-11) +********************** Features - 20.0.2 -~~~~~~~~~~~~~~~~~ +================= + - Print out a one line message about the created virtual environment when no :option:`verbose` is set, this can now be - silenced to get back the original behavior via the :option:`quiet` flag - by :user:`pradyunsg`. (`#1557 `_) -- Allow virtualenv's app data cache to be overridden by ``VIRTUALENV_OVERRIDE_APP_DATA`` - by :user:`asottile`. (`#1559 `_) -- Passing in the virtual environment name/path is now required (no longer defaults to ``venv``) - by :user:`gaborbernat`. (`#1568 `_) + silenced to get back the original behavior via the :option:`quiet` flag - by :user:`pradyunsg`. (`#1557 + `_) +- Allow virtualenv's app data cache to be overridden by ``VIRTUALENV_OVERRIDE_APP_DATA`` - by :user:`asottile`. (`#1559 + `_) +- Passing in the virtual environment name/path is now required (no longer defaults to ``venv``) - by + :user:`gaborbernat`. (`#1568 `_) - Add a CLI flag :option:`with-traceback` that allows displaying the stacktrace of the virtualenv when a failure occurs - by :user:`gaborbernat`. (`#1572 `_) Bugfixes - 20.0.2 -~~~~~~~~~~~~~~~~~ +================= + - Support long path names for generated virtual environment console entry points (such as ``pip``) when using the ``app-data`` :option:`seeder` - by :user:`gaborbernat`. (`#997 `_) - Improve python discovery mechanism: @@ -1560,71 +2801,88 @@ Bugfixes - 20.0.2 - beside the prefix folder also try with the platform dependent binary folder within that, by :user:`gaborbernat`. (`#1545 `_) + - When copying (either files or trees) do not copy the permission bits, last access time, last modification time, and - flags as access to these might be forbidden (for example in case of the macOs Framework Python) and these are not needed - for the user to use the virtual environment - by :user:`gaborbernat`. (`#1561 `_) + flags as access to these might be forbidden (for example in case of the macOs Framework Python) and these are not + needed for the user to use the virtual environment - by :user:`gaborbernat`. (`#1561 + `_) - While discovering a python executables interpreters that cannot be queried are now displayed with info level rather than warning, so now they're no longer shown by default (these can be just executables to which we don't have access - or that are broken, don't warn if it's not the target Python we want) - by :user:`gaborbernat`. (`#1574 `_) + or that are broken, don't warn if it's not the target Python we want) - by :user:`gaborbernat`. (`#1574 + `_) - The ``app-data`` :option:`seeder` no longer symlinks the packages on UNIX and copies on Windows. Instead by default - always copies, however now has the :option:`symlink-app-data` flag allowing users to request this less robust but faster - method - by :user:`gaborbernat`. (`#1575 `_) + always copies, however now has the :option:`symlink-app-data` flag allowing users to request this less robust but + faster method - by :user:`gaborbernat`. (`#1575 `_) Improved Documentation - 20.0.2 -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -- Add link to the `legacy documentation `_ for the changelog by :user:`jezdez`. (`#1547 `_) -- Fine tune the documentation layout: default width of theme, allow tables to wrap around, soft corners for code snippets - - by :user:`pradyunsg`. (`#1548 `_) +=============================== +- Add link to the `legacy documentation `_ for the changelog by :user:`jezdez`. + (`#1547 `_) +- Fine tune the documentation layout: default width of theme, allow tables to wrap around, soft corners for code + snippets - by :user:`pradyunsg`. (`#1548 `_) -v20.0.1 (2020-02-10) --------------------- +********************** + v20.0.1 (2020-02-10) +********************** Features - 20.0.1 -~~~~~~~~~~~~~~~~~ -- upgrade embedded setuptools to ``45.2.0`` from ``45.1.0`` for Python ``3.4+`` - by :user:`gaborbernat`. (`#1554 `_) +================= + +- upgrade embedded setuptools to ``45.2.0`` from ``45.1.0`` for Python ``3.4+`` - by :user:`gaborbernat`. (`#1554 + `_) Bugfixes - 20.0.1 -~~~~~~~~~~~~~~~~~ -- Virtual environments created via relative path on Windows creates bad console executables - by :user:`gaborbernat`. (`#1552 `_) -- Seems sometimes venvs created set their base executable to themselves; we accept these without question, so we handle - virtual environments as system pythons causing issues - by :user:`gaborbernat`. (`#1553 `_) +================= +- Virtual environments created via relative path on Windows creates bad console executables - by :user:`gaborbernat`. + (`#1552 `_) +- Seems sometimes venvs created set their base executable to themselves; we accept these without question, so we handle + virtual environments as system pythons causing issues - by :user:`gaborbernat`. (`#1553 + `_) -v20.0.0. (2020-02-10) ---------------------- +*********************** + v20.0.0. (2020-02-10) +*********************** Improved Documentation - 20.0.0. -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +================================ + - Fixes typos, repeated words and inconsistent heading spacing. Rephrase parts of the development documentation and CLI - documentation. Expands shorthands like ``env var`` and ``config`` to their full forms. Uses descriptions from respective - documentation, for projects listed in ``related links`` - by :user:`pradyunsg`. (`#1540 `_) + documentation. Expands shorthands like ``env var`` and ``config`` to their full forms. Uses descriptions from + respective documentation, for projects listed in ``related links`` - by :user:`pradyunsg`. (`#1540 + `_) -v20.0.0b2 (2020-02-04) ----------------------- +************************ + v20.0.0b2 (2020-02-04) +************************ Features - 20.0.0b2 -~~~~~~~~~~~~~~~~~~~ +=================== + - Improve base executable discovery mechanism: - print at debug level why we refuse some candidates, - when no candidates match exactly, instead of hard failing fallback to the closest match where the priority of - matching attributes is: python implementation, major version, minor version, architecture, patch version, - release level and serial (this is to facilitate things to still work when the OS upgrade replace/upgrades the system - python with a never version, than what the virtualenv host python was created with), + matching attributes is: python implementation, major version, minor version, architecture, patch version, release + level and serial (this is to facilitate things to still work when the OS upgrade replace/upgrades the system python + with a never version, than what the virtualenv host python was created with), - always resolve system_executable information during the interpreter discovery, and the discovered environment is the - system interpreter instead of the venv/virtualenv (this happened before lazily the first time we accessed, and caused - reporting that the created virtual environment is of type of the virtualenv host python version, instead of the - system pythons version - these two can differ if the OS upgraded the system python underneath and the virtualenv + system interpreter instead of the venv/virtualenv (this happened before lazily the first time we accessed, and + caused reporting that the created virtual environment is of type of the virtualenv host python version, instead of + the system pythons version - these two can differ if the OS upgraded the system python underneath and the virtualenv host was created via copy), by :user:`gaborbernat`. (`#1515 `_) -- Generate ``bash`` and ``fish`` activators on Windows too (as these can be available with git bash, cygwin or mysys2) - - by :user:`gaborbernat`. (`#1527 `_) -- Upgrade the bundled ``wheel`` package from ``0.34.0`` to ``0.34.2`` - by :user:`gaborbernat`. (`#1531 `_) + +- Generate ``bash`` and ``fish`` activators on Windows too (as these can be available with git bash, cygwin or mysys2) - + by :user:`gaborbernat`. (`#1527 `_) +- Upgrade the bundled ``wheel`` package from ``0.34.0`` to ``0.34.2`` - by :user:`gaborbernat`. (`#1531 + `_) Bugfixes - 20.0.0b2 -~~~~~~~~~~~~~~~~~~~ +=================== + - Bash activation script should have no extensions instead of ``.sh`` (this fixes the :pypi:`virtualenvwrapper` integration) - by :user:`gaborbernat`. (`#1508 `_) - Show less information when we run with a single verbosity (``-v``): @@ -1634,32 +2892,42 @@ Bugfixes - 20.0.0b2 - for the ``app-data`` seeder do not show the type of lock, only the path to the app data directory, By :user:`gaborbernat`. (`#1510 `_) + - Fixed cannot discover a python interpreter that has already been discovered under a different path (such is the case - when we have multiple symlinks to the same interpreter) - by :user:`gaborbernat`. (`#1512 `_) -- Support relative paths for ``-p`` - by :user:`gaborbernat`. (`#1514 `_) -- Creating virtual environments in parallel fail with cannot acquire lock within app data - by :user:`gaborbernat`. (`#1516 `_) -- pth files were not processed under Debian CPython2 interpreters - by :user:`gaborbernat`. (`#1517 `_) + when we have multiple symlinks to the same interpreter) - by :user:`gaborbernat`. (`#1512 + `_) +- Support relative paths for ``-p`` - by :user:`gaborbernat`. (`#1514 + `_) +- Creating virtual environments in parallel fail with cannot acquire lock within app data - by :user:`gaborbernat`. + (`#1516 `_) +- pth files were not processed under Debian CPython2 interpreters - by :user:`gaborbernat`. (`#1517 + `_) - Fix prompt not displayed correctly with upcoming fish 3.10 due to us not preserving ``$pipestatus`` - by :user:`krobelus`. (`#1530 `_) - Stable order within ``pyenv.cfg`` and add ``include-system-site-packages`` only for creators that reference a global Python - by user:`gaborbernat`. (`#1535 `_) Improved Documentation - 20.0.0b2 -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -- Create the first iteration of the new documentation - by :user:`gaborbernat`. (`#1465 `_) -- Project readme is now of type MarkDown instead of reStructuredText - by :user:`gaborbernat`. (`#1531 `_) +================================= +- Create the first iteration of the new documentation - by :user:`gaborbernat`. (`#1465 + `_) +- Project readme is now of type MarkDown instead of reStructuredText - by :user:`gaborbernat`. (`#1531 + `_) -v20.0.0b1 (2020-01-28) ----------------------- +************************ + v20.0.0b1 (2020-01-28) +************************ -* First public release of the rewrite. Everything is brand new and just added. -* ``--download`` defaults to ``False`` -* No longer replaces builtin ``site`` module with `custom version baked within virtualenv code itself `_. A simple shim module is used to fix up things on Python 2 only. +- First public release of the rewrite. Everything is brand new and just added. +- ``--download`` defaults to ``False`` +- No longer replaces builtin ``site`` module with `custom version baked within virtualenv code itself + `_. A simple shim module is used to fix up + things on Python 2 only. .. warning:: - The current virtualenv is the second iteration of implementation. From version ``0.8`` all the way to ``16.7.9`` - we numbered the first iteration. Version ``20.0.0b1`` is a complete rewrite of the package, and as such this release - history starts from there. The old changelog is still available in the - `legacy branch documentation `_. + The current virtualenv is the second iteration of implementation. From version ``0.8`` all the way to ``16.7.9`` we + numbered the first iteration. Version ``20.0.0b1`` is a complete rewrite of the package, and as such this release + history starts from there. The old changelog is still available in the `legacy branch documentation + `_. diff --git a/docs/changelog/examples.rst b/docs/changelog/examples.rst index eda79eb70..86257006a 100644 --- a/docs/changelog/examples.rst +++ b/docs/changelog/examples.rst @@ -1,15 +1,20 @@ .. examples for changelog entries adding to your Pull Requests -file ``544.doc.rst``:: +file ``544.doc.rst``: + +:: explain everything much better - by :user:`passionate_technicalwriter`. -file ``544.feature.rst``:: +file ``544.feature.rst``: + +:: ``tox --version`` now shows information about all registered plugins - by :user:`obestwalter`. +file ``571.bugfix.rst``: -file ``571.bugfix.rst``:: +:: ``skip_install`` overrides ``usedevelop`` (``usedevelop`` is an option to choose the installation type if the package is installed and ``skip_install`` determines if it should be diff --git a/docs/changelog/template.jinja2 b/docs/changelog/template.jinja2 index bb88fa2c2..9de49f34a 100644 --- a/docs/changelog/template.jinja2 +++ b/docs/changelog/template.jinja2 @@ -1,18 +1,18 @@ -{% set top_underline = underlines[0] %} {% if versiondata.name %} -v{{ versiondata.version }} ({{ versiondata.date }}) -{{ top_underline * ((versiondata.version + versiondata.date)|length + 4)}} +{% set version_title = "v" + versiondata.version + " (" + versiondata.date + ")" %} {% else %} -{{ versiondata.version }} ({{ versiondata.date }}) -{{ top_underline * ((versiondata.version + versiondata.date)|length + 3)}} +{% set version_title = versiondata.version + " (" + versiondata.date + ")" %} {% endif %} +{{ top_underline * (version_title|length + 2) }} + {{ version_title }} +{{ top_underline * (version_title|length + 2) }} {% for section, _ in sections.items() %} -{% set underline = underlines[1] %} {% if sections[section] %} {% for category, val in definitions.items() if category in sections[section]%} {{ definitions[category]['name'] }} - {{ versiondata.version }} -{{ underline * ((definitions[category]['name'] + versiondata.version)|length + 3)}} +{{ underlines[0] * ((definitions[category]['name'] + versiondata.version)|length + 3) }} + {% if definitions[category]['showcontent'] %} {% for text, values in sections[section][category].items() %} - {{ text }} ({{ values|join(', ') }}) diff --git a/docs/cli_interface.rst b/docs/cli_interface.rst deleted file mode 100644 index 27ea231ef..000000000 --- a/docs/cli_interface.rst +++ /dev/null @@ -1,137 +0,0 @@ -CLI interface -============= - -.. _cli_flags: - -CLI flags -~~~~~~~~~ - -``virtualenv`` is primarily a command line application. - -It modifies the environment variables in a shell to create an isolated Python environment, so you'll need to have a -shell to run it. You can type in ``virtualenv`` (name of the application) followed by flags that control its -behavior. All options have sensible defaults, and there's one required argument: the name/path of the virtual -environment to create. The default values for the command line options can be overridden via the -:ref:`conf_file` or :ref:`env_vars`. Environment variables takes priority over the configuration file values -(``--help`` will show if a default comes from the environment variable as the help message will end in this case -with environment variables or the configuration file). - -The options that can be passed to virtualenv, along with their default values and a short description are listed below. - -:command:`virtualenv [OPTIONS]` - -.. table_cli:: - :module: virtualenv.run - :func: build_parser_only - -Discovery options -~~~~~~~~~~~~~~~~~ - -Understanding Interpreter Discovery: ``--python`` vs. ``--try-first-with`` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -You can control which Python interpreter ``virtualenv`` selects using the ``--python`` and ``--try-first-with`` flags. -To avoid confusion, it's best to think of them as the "rule" and the "hint". - -**``--python ``: The Rule** - -This flag sets the mandatory requirements for the interpreter. The ```` can be: - -- **A version string** (e.g., ``python3.8``, ``pypy3``). ``virtualenv`` will search for any interpreter that matches this version. -- **An absolute path** (e.g., ``/usr/bin/python3.8``). This is a *strict* requirement. Only the interpreter at this exact path will be used. If it does not exist or is not a valid interpreter, creation will fail. - -**``--try-first-with ``: The Hint** - -This flag provides a path to a Python executable to check *before* ``virtualenv`` performs its standard search. This can speed up discovery or help select a specific interpreter when multiple versions exist on your system. - -**How They Work Together** - -``virtualenv`` will only use an interpreter from ``--try-first-with`` if it **satisfies the rule** from the ``--python`` flag. The ``--python`` rule always wins. - -**Examples:** - -1. **Hint does not match the rule:** - - .. code-block:: bash - - virtualenv --python python3.8 --try-first-with /usr/bin/python3.10 my-env - - - **Result:** ``virtualenv`` first inspects ``/usr/bin/python3.10``. It sees this does not match the ``python3.8`` rule and **rejects it**. It then proceeds with its normal search to find a ``python3.8`` interpreter elsewhere. - -2. **Hint does not match a strict path rule:** - - .. code-block:: bash - - virtualenv --python /usr/bin/python3.8 --try-first-with /usr/bin/python3.10 my-env - - - **Result:** The rule is strictly ``/usr/bin/python3.8``. ``virtualenv`` checks the ``/usr/bin/python3.10`` hint, sees the path doesn't match, and **rejects it**. It then moves on to test ``/usr/bin/python3.8`` and successfully creates the environment. - -This approach ensures that the behavior is predictable and that ``--python`` remains the definitive source of truth for the user's intent. - - -Defaults -~~~~~~~~ - -.. _conf_file: - -Configuration file -^^^^^^^^^^^^^^^^^^ - -Unless ``VIRTUALENV_CONFIG_FILE`` is set, virtualenv looks for a standard ``virtualenv.ini`` configuration file. -The exact location depends on the operating system you're using, as determined by :pypi:`platformdirs` application -configuration definition. It can be overridden by setting the ``VIRTUALENV_CONFIG_FILE`` environment variable. -The configuration file location is printed as at the end of the output when ``--help`` is passed. - -The keys of the settings are derived from the command line option (left strip the ``-`` characters, and replace ``-`` -with ``_``). Where multiple flags are available first found wins (where order is as it shows up under the ``--help``). - -For example, :option:`--python ` would be specified as: - -.. code-block:: ini - - [virtualenv] - python = /opt/python-3.8/bin/python - -Options that take multiple values, like :option:`extra-search-dir` can be specified as: - -.. code-block:: ini - - [virtualenv] - extra_search_dir = - /path/to/dists - /path/to/other/dists - -.. _env_vars: - -Environment Variables -^^^^^^^^^^^^^^^^^^^^^ - -Default values may be also specified via environment variables. The keys of the settings are derived from the -command line option (left strip the ``-`` characters, and replace ``-`` with ``_``, finally capitalize the name). Where -multiple flags are available first found wins (where order is as it shows up under the ``--help``). - -For example, to use a custom Python binary, instead of the one virtualenv is run with, you can set the environment -variable ``VIRTUALENV_PYTHON`` like: - -.. code-block:: console - - env VIRTUALENV_PYTHON=/opt/python-3.8/bin/python virtualenv - -Where the option accepts multiple values, for example for :option:`python` or -:option:`extra-search-dir`, the values can be separated either by literal -newlines or commas. Newlines and commas can not be mixed and if both are -present only the newline is used for separating values. Examples for multiple -values: - - -.. code-block:: console - - env VIRTUALENV_PYTHON=/opt/python-3.8/bin/python,python3.8 virtualenv - env VIRTUALENV_EXTRA_SEARCH_DIR=/path/to/dists\n/path/to/other/dists virtualenv - -The equivalent CLI-flags based invocation for the above examples would be: - -.. code-block:: console - - virtualenv --python=/opt/python-3.8/bin/python --python=python3.8 - virtualenv --extra-search-dir=/path/to/dists --extra-search-dir=/path/to/other/dists diff --git a/docs/conf.py b/docs/conf.py index f02c8a7cb..1594a9312 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,6 +1,7 @@ from __future__ import annotations -import subprocess +import json +import os import sys from datetime import datetime, timezone from pathlib import Path @@ -11,14 +12,29 @@ name = "virtualenv" version = ".".join(__version__.split(".")[:2]) release = __version__ -copyright = f"2007-{datetime.now(tz=timezone.utc).year}, {company}, PyPA" # noqa: A001 +copyright = f"2007-{datetime.now(tz=timezone.utc).year}, {company}, PyPA" # ruff:ignore[builtin-variable-shadowing] extensions = [ + "sphinx_llm.txt", "sphinx.ext.autodoc", "sphinx.ext.autosectionlabel", "sphinx.ext.extlinks", + "sphinx.ext.intersphinx", + "sphinx_autodoc_typehints", + "sphinx_copybutton", + "sphinx_inline_tabs", + "sphinxcontrib.mermaid", + "sphinxcontrib.towncrier.ext", ] +intersphinx_mapping = { + "python": ("https://docs.python.org/3", None), +} + +towncrier_draft_autoversion_mode = "draft" +towncrier_draft_include_empty = True +towncrier_draft_working_directory = Path(__file__).parent.parent + templates_path = [] unused_docs = [] source_suffix = ".rst" @@ -33,7 +49,21 @@ html_theme = "furo" html_title, html_last_updated_fmt = project, datetime.now(tz=timezone.utc).isoformat() pygments_style, pygments_dark_style = "sphinx", "monokai" -html_static_path, html_css_files = ["_static"], ["custom.css"] +html_static_path = ["_static"] +html_css_files = ["custom.css"] +html_js_files = ["mermaid-normalize.js", "rtd-search.js"] +# sphinxcontrib-mermaid defaults to whichever Mermaid its release shipped with; pin it so the upgrade workflow bumps it +mermaid_version = json.loads((Path(__file__).parents[1] / "tasks" / "ci-tools.json").read_text(encoding="utf-8"))[ + "mermaid" +]["version"] +# favicon is a simplified mark, not a vector copy of the logo PNG; see development.rst "Logo and branding" +html_favicon = "_static/virtualenv.svg" +html_theme_options = { + "light_logo": "virtualenv.png", + "dark_logo": "virtualenv.png", + "sidebar_hide_name": True, +} +html_show_sourcelink = False autoclass_content = "both" # Include __init__ in class documentation autodoc_member_order = "bysource" @@ -47,27 +77,22 @@ } -def setup(app): - here = Path(__file__).parent - root, exe = here.parent, Path(sys.executable) - towncrier = exe.with_name(f"towncrier{exe.suffix}") - cmd = [str(towncrier), "build", "--draft", "--version", "NEXT"] - new = subprocess.check_output(cmd, cwd=root, text=True, stderr=subprocess.DEVNULL, encoding="UTF-8") - (root / "docs" / "_draft.rst").write_text("" if "No significant changes" in new else new, encoding="UTF-8") - - # the CLI arguments are dynamically generated +def setup(app) -> None: doc_tree = Path(app.doctreedir) - cli_interface_doctree = doc_tree / "cli_interface.doctree" - if cli_interface_doctree.exists(): - cli_interface_doctree.unlink() + for name in ("cli_interface", "reference/cli"): + doctree = doc_tree / f"{name}.doctree" + if doctree.exists(): + doctree.unlink() here = Path(__file__).parent if str(here) not in sys.path: sys.path.append(str(here)) - # noinspection PyUnresolvedReferences - from render_cli import CliTable, literal_data # noqa: PLC0415 + from render_cli import CliTable, literal_data # ruff:ignore[import-outside-top-level] app.add_css_file("custom.css") app.add_directive(CliTable.name, CliTable) app.add_role("literal_data", literal_data) + + +markdown_http_base = os.environ.get("READTHEDOCS_CANONICAL_URL", "") diff --git a/docs/development.rst b/docs/development.rst index 941a8b5c9..8968ab28f 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -1,38 +1,38 @@ -Development -=========== - -Getting started ---------------- +############# + Development +############# +***************** + Getting started +***************** ``virtualenv`` is a volunteer maintained open source project and we welcome contributions of all forms. The sections below will help you get started with development, testing, and documentation. We’re pleased that you are interested in working on virtualenv. This document is meant to get you setup to work on virtualenv and to act as a guide and reference -to the development setup. If you face any issues during this process, please -`open an issue `_ about it on -the issue tracker. +to the development setup. If you face any issues during this process, please `open an issue +`_ about it on the issue +tracker. Setup -~~~~~ +===== virtualenv is a command line application written in Python. To work on it, you'll need: - **Source code**: available on `GitHub `_. You can use ``git`` to clone the - repository: + repository: .. code-block:: console git clone https://github.com/pypa/virtualenv cd virtualenv -- **Python interpreter**: We recommend using ``CPython``. You can use - `this guide `_ to set it up. - +- **Python interpreter**: We recommend using ``CPython``. You can use `this guide + `_ to set it up. - :pypi:`tox`: to automatically get the projects development dependencies and run the test suite. We recommend installing it using `pipx `_. Running from source tree -~~~~~~~~~~~~~~~~~~~~~~~~ +======================== The easiest way to do this is to generate the development tox environment, and then invoke virtualenv from under the ``.tox/dev`` folder @@ -44,10 +44,10 @@ The easiest way to do this is to generate the development tox environment, and t .tox/dev/Scripts/virtualenv # on Windows Running tests -~~~~~~~~~~~~~ +============= -virtualenv's tests are written using the :pypi:`pytest` test framework. :pypi:`tox` is used to automate the setup -and execution of virtualenv's tests. +virtualenv's tests are written using the :pypi:`pytest` test framework. :pypi:`tox` is used to automate the setup and +execution of virtualenv's tests. To run tests locally execute: @@ -55,13 +55,12 @@ To run tests locally execute: tox -e py -This will run the test suite for the same Python version as under which ``tox`` is installed. Alternatively you can -specify a specific version of python by using the ``pyNN`` format, such as: ``py38``, ``pypy3``, etc. +This runs the test suite on Python 3.15. Select another supported interpreter with its environment name, such as +``3.9``, ``3.14`` or ``pypy3.11``. -``tox`` has been configured to forward any additional arguments it is given to ``pytest``. -This enables the use of pytest's -`rich CLI `_. As an example, you can -select tests using the various ways that pytest provides: +``tox`` has been configured to forward any additional arguments it is given to ``pytest``. This enables the use of +pytest's `rich CLI `_. As an example, you +can select tests using the various ways that pytest provides: .. code-block:: console @@ -71,11 +70,19 @@ select tests using the various ways that pytest provides: tox -e py -- -k "test_extra" Some tests require additional dependencies to be run, such is the various shell activators (``bash``, ``fish``, -``powershell``, etc). These tests will automatically be skipped if these are not present, note however that in CI -all tests are run; so even if all tests succeed locally for you, they may still fail in the CI. +``powershell``, etc). These tests will automatically be skipped if these are not present, note however that in CI all +tests are run; so even if all tests succeed locally for you, they may still fail in the CI. + +The tests under ``tests/tasks`` cover the build and CI tooling, ``hatch_build.py`` and the scripts in ``tasks/``. Their +outcome does not depend on the interpreter, so the interpreter environments skip them and one environment runs them on +CPython 3.15: + +.. code-block:: console + + tox -e tasks Running linters -~~~~~~~~~~~~~~~ +=============== virtualenv uses :pypi:`pre-commit` for managing linting of the codebase. ``pre-commit`` performs various checks on all files in virtualenv and uses tools that help follow a consistent code style within the codebase. To use linters locally, @@ -90,8 +97,42 @@ run: Avoid using ``# noqa`` comments to suppress linter warnings - wherever possible, warnings should be fixed instead. ``# noqa`` comments are reserved for rare cases where the recommended style causes severe readability problems. +Type checking +============= + +virtualenv ships a :PEP:`561` ``py.typed`` marker and has comprehensive type annotations across the entire codebase. +This means downstream consumers and type checkers automatically recognize virtualenv as an inline-typed package. + +All new code **must** include complete type annotations for function parameters and return types. To verify annotations +locally, run: + +.. code-block:: console + + tox -e type + +This uses `ty `_ (Astral's Rust-based type checker) to validate annotations against Python +3.15. A second environment checks compatibility with the minimum supported version: + +.. code-block:: console + + tox -e type-3.9 + +Both environments validate that annotations are consistent and correct. + +Annotation guidelines +--------------------- + +- Use ``from __future__ import annotations`` at the top of every module (enforced by ruff's ``required-imports`` + setting). +- Place imports that are only needed for type checking inside an ``if TYPE_CHECKING:`` block to avoid runtime overhead. +- Ruff's ``ANN`` rules are enabled. ``ANN401`` (``typing.Any``) is suppressed on a case-by-case basis with inline ``# + noqa: ANN401`` comments where ``Any`` is genuinely required (e.g. serialization, dynamic dispatch). +- Prefer concrete types over ``Any``. Use ``Union`` / ``|`` for nullable or multi-type parameters. +- When a type error is genuinely unfixable (e.g. third-party library limitations), suppress it with an inline ``# ty: + ignore[rule-name]`` comment and a brief justification. + Building documentation -~~~~~~~~~~~~~~~~~~~~~~ +====================== virtualenv's documentation is built using :pypi:`Sphinx`. The documentation is written in reStructuredText. To build it locally, run: @@ -103,24 +144,95 @@ locally, run: The built documentation can be found in the ``.tox/docs_out`` folder and may be viewed by opening ``index.html`` within that folder. +Logo and branding +================= + +Two logo files live in ``docs/_static`` and serve different roles, so both are kept on purpose: + +- ``virtualenv.png`` is the official logo — the snakes wrapped around a terminal, with the wordmark. It is the site logo + (``light_logo``/``dark_logo`` in ``docs/conf.py``) and the one to use wherever a virtualenv logo is needed. +- ``virtualenv.svg`` is a simplified mark used only as the browser favicon (``html_favicon``), where the detailed PNG + would be illegible at 16×16. It is not a vector copy of the PNG. + +.. _release: + Release -~~~~~~~ +======= + +virtualenv's release schedule is tied to ``pip`` and ``setuptools``. We bundle the latest version of these libraries so +each time there's a new version of any of these, there will be a new virtualenv release shortly afterwards (we usually +wait just a few days to avoid pulling in any broken releases). + +Performing a release +-------------------- + +A full release publishes to `PyPI `_, creates a `GitHub Release +`_ with the zipapp and SBOMs attached, and updates `get-virtualenv +`_ so that ``https://bootstrap.pypa.io/virtualenv.pyz`` serves the new version. + +Version bumping +^^^^^^^^^^^^^^^ + +The ``--version`` argument to ``tox r -e release`` controls the version. It defaults to ``auto``, which inspects the +``docs/changelog`` directory: if any ``*.feature.rst`` or ``*.removal.rst`` fragments exist, the minor version is +bumped, otherwise the patch version is bumped. You can also pass ``major``, ``minor``, or ``patch`` explicitly. + +Both methods produce identical results: a release commit and tag on ``main``. Pushing the tag triggers the `Release +workflow `_ which builds the sdist, wheel, and +zipapp, publishes to PyPI via trusted publisher, creates a `GitHub Release +`_ with the zipapp and SBOMs attached, and updates `get-virtualenv +`_. A failed publish needs the recovery procedure below. + +**Via GitHub Actions (recommended)** + +1. Go to the `Pre-release workflow `_ on GitHub. +2. Click **Run workflow** and select the bump type (``auto``, ``major``, ``minor``, or ``patch``). + +**Locally** + +.. code-block:: console -virtualenv's release schedule is tied to ``pip`` and ``setuptools``. We bundle the latest version of these -libraries so each time there's a new version of any of these, there will be a new virtualenv release shortly afterwards -(we usually wait just a few days to avoid pulling in any broken releases). + tox r -e release -Contributing -------------- +Pass ``--version `` to override the default ``auto`` behavior (e.g. ``--version minor``). + +Recovering a partial publication +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A failure after the PyPI upload can leave a published package without its GitHub release or bootstrap update. Keep the +release commit and tag: users may have installed that version, and PyPI `does not allow filename reuse +`_. Resetting the tip of ``main`` can also remove an unrelated commit that a +maintainer merged during publication. + +Before retrying, inspect the failed job, the files on PyPI, the GitHub release assets and +``get-virtualenv/public/version.txt``. Compare their hashes with the artifacts from the failed workflow run. Preserve +those artifacts and logs. A timeout does not prove that an upload failed. + +Complete missing publication steps with the original verified artifacts. Do not rerun the full publish job after PyPI +accepted its upload; the duplicate upload can fail before reaching the remaining destinations. If the artifacts need a +change, publish a new version and consider yanking the broken release. Use a normal reviewed commit for any bootstrap +correction, without force-pushing either repository. Follow ``.github/INCIDENT_RESPONSE.md`` if there is evidence of +tampering or compromised credentials. + +The release workflow verifies the published wheel, sdist and zipapp after publication. It checks the PyPI attestations +and compares the distributions with the build artifacts, verifies the zipapp's release workflow, tag and commit, checks +that the ``virtualenv.cdx.json`` and ``virtualenv.spdx.json`` release assets match the wheel and the attested SBOMs, +then creates an environment from each distribution. This job has no publishing credentials. A verification failure does +not undo publication: inspect the mismatch or creation failure before retrying only the verification job. Do not rerun +publication to repair a verification failure. + +************** + Contributing +************** Submitting pull requests -~~~~~~~~~~~~~~~~~~~~~~~~ +======================== Submit pull requests against the ``main`` branch, providing a good description of what you're doing and why. You must have legal permission to distribute any code you contribute to virtualenv and it must be available under the MIT -License. Provide tests that cover your changes and run the tests locally first. virtualenv -:ref:`supports ` multiple Python versions and operating systems. Any pull request must -consider and work on all these platforms. +License. Provide tests that cover your changes and run the tests locally first. virtualenv :ref:`supports +` multiple Python versions and operating systems. Any pull request must consider and work on +all these platforms. Pull Requests should be small to facilitate review. Keep them self-contained, and limited in scope. `Studies have shown `_ that review quality falls off as patch size @@ -132,25 +244,155 @@ Additionally, avoid including "cosmetic" changes to code that is unrelated to yo PR more difficult. Examples include re-flowing text in comments or documentation, or addition or removal of blank lines or whitespace within lines. Such changes can be made separately, as a "formatting cleanup" PR, if needed. +Code review +=========== + +A maintainer reviews each pull request before it merges into ``main`` and checks that: + +- the change does what its description says, and a test covers it; +- the change works on each Python version and operating system that :ref:`compatibility-requirements` lists; +- a news fragment describes each user-facing change, and the documentation matches the new behavior; +- the command line, the Python API and the plugin API stay compatible, or a deprecation or removal fragment announces + the break; +- a change to activation scripts, ``pyvenv.cfg``, wheel downloads, app-data or the release workflows names in its + description the outside input it handles, and updates the `threat model + `_ when it hits an update + trigger; +- a new or bumped dependency meets the :ref:`licensing policy `. + +The ``main`` ruleset enforces the rest. A pull request merges only after four checks pass: ``✅ all checks pass``, which +gathers the test matrix, the type checks, the docs build, the zipapp and SBOM checks and the CodeQL query tests from +``check.yaml``; ``🔍 dependency review``; ``pre-commit.ci - pr``; and the Read the Docs build. The ruleset also blocks a +merge while CodeQL reports an error, or a security alert of high severity or above. + +The ruleset asks for one approval from someone other than the last person to push. Repository admins can bypass that +rule, and the maintainers merge their own pull requests this way once the required checks pass. The `threat model +`_ records the +risk of a single reviewer. + +AI-assisted contributions +========================= + +You may use AI tools (code assistants, chat models, agents) to help write a contribution, under these conditions: + +- You are the author. Read, understand and test everything you submit; you are responsible for it as if you had typed it + yourself, including its licensing. +- Keep a human in the loop. Do not open pull requests or issues, or post review comments, that you have not read. Fully + automated submissions are closed without review. +- Do not paste in code whose license is unknown or incompatible with the MIT License, whatever produced it. + +Maintainers may use AI tools to help triage and review. A review comment posted under a maintainer's name has been read +and endorsed by that maintainer. + +Dependency policy +================= + +virtualenv lists its runtime dependencies under ``[project.dependencies]`` in ``pyproject.toml``, and its development +and release tools in dependency groups and ``tox.toml``. Adding a runtime dependency is a maintainer decision under the +`Licensing policy`_ below. + +The project obtains and updates each kind of dependency this way: + +- Installers fetch runtime dependencies from PyPI within the ranges ``pyproject.toml`` allows. +- The daily ``upgrade.yaml`` workflow runs ``tox r -e upgrade``, which downloads new ``pip`` and ``setuptools`` wheels + from PyPI into ``src/virtualenv/seed/wheels/embed`` and records their SHA-256 in ``BUNDLE_SHA256``. The same workflow + recompiles ``pylock.zipapp.toml``, which pins each distribution the zipapp bundles by version and hash. It leaves out + releases younger than seven days and opens a pull request with the result. +- Dependabot bumps the GitHub Actions, which the repository requires to be pinned to a commit SHA, every week after a + seven-day cooldown. pre-commit.ci bumps the pre-commit hooks, which ``.pre-commit-config.yaml`` freezes to commit + SHAs. ``upgrade.yaml`` bumps the CI test tools every Monday. + +We track dependencies through the CycloneDX SBOM in every wheel and the SBOMs attached to each release, which +:doc:`reference/release-artifacts` describes, and through Dependabot alerts, which cover the pip and GitHub Actions +manifests. + +Software composition analysis findings follow these thresholds: + +- ``dependency-review.yaml`` fails a pull request that adds a dependency with a known advisory of any severity, in + runtime or development scope. The ``main`` ruleset requires that check. +- A shipped package whose license falls outside ``ALLOWED_LICENSES`` fails the SBOM check in ``tox r -e readme`` and + ``tox r -e zipapp``, which the required ``✅ all checks pass`` check runs. We accept no exceptions. +- An open Dependabot alert blocks the next release. The maintainer who cuts the release fixes it first, or dismisses it + with the reason virtualenv does not reach the vulnerable code. +- An embedded wheel with no fixed version for a supported Python version stays, and + :ref:`reference/release-artifacts:Embedded wheel advisories` lists its advisories and the way around them. + +Licensing policy +================ + +virtualenv is distributed under the MIT License, and everything in the repository must be compatible with it: + +- Contributions are accepted under the MIT License only; you must have the right to license what you submit. +- What virtualenv ships must use a permissive license from ``ALLOWED_LICENSES`` in ``tasks/license_policy.py``. That + covers virtualenv itself, every distribution the zipapp bundles, and the embedded ``pip`` and ``setuptools`` wheels + under ``src/virtualenv/seed/wheels/embed``, which keep their own MIT licenses and ship unchanged. The packages those + wheels vendor follow the terms of their wheel. The Unlicense entry covers ``filelock`` 3.19.1 and older, which + virtualenv installs on Python 3.9 and bundles in the zipapp. Copyleft licenses (GPL, LGPL, AGPL, MPL) stay out. + Development and release tools carry no such restriction. +- ``tasks/validate_sbom.py`` enforces the policy on the SBOMs of the built wheel and zipapp, in ``tox r -e readme`` and + ``tox r -e zipapp``. An SPDX expression passes when each ``OR`` offers an allowed license and every ``AND`` operand is + allowed; a ``WITH`` exception leaves the verdict to the license it modifies. A package that declares its license by + name alone fails the check until ``_CLASSIFIER_LICENSES`` in ``hatch_build.py`` maps the name to an SPDX id. +- ``.github/workflows/dependency-review.yaml`` fails a pull request that adds a dependency with a known vulnerability to + any manifest, release tooling lock files included. +- Adding a runtime dependency or bumping an embedded wheel is a maintainer decision; checking the license of the new + version is part of that review. +- ``hatch_build.py`` writes a `CycloneDX `_ SBOM into every wheel at + ``.dist-info/sboms/virtualenv.cdx.json``. It reads the ``BUNDLE_SUPPORT``/``BUNDLE_SHA256`` tables that back the + embedded wheels above, so a wheel bump updates it with no extra step. ``tasks/cyclonedx_to_spdx.py`` renders that + document as SPDX 2.3 with the standard library alone. ``tox r -e readme`` builds a wheel and validates its SBOM and + the SPDX rendering; the release workflow runs ``tox r -e spdx`` to write ``virtualenv.spdx.json`` from + ``virtualenv.cdx.json`` and check it with ``pyspdxtools``. Both environments run on Python 3.15 with a Beartype + prerelease that fixes its `removed typing API import `_. +- ``tasks/zipapp_sbom.py`` describes a built ``virtualenv.pyz`` from the archive itself: virtualenv, the embedded + wheels, and each bundled dependency with the Python versions that load it, down to a SHA-256 per file. It reuses the + helpers in ``hatch_build.py``, writes ``virtualenv.pyz.cdx.json`` next to the zipapp and adds the same file to the + archive root. ``tox r -e zipapp`` builds the zipapp, runs the script, and has ``tasks/validate_sbom.py`` check that + the SBOM lists every file in the archive with its hash. + +:doc:`reference/release-artifacts` describes every file, SBOM and attestation a release publishes, and +:doc:`how-to/verify-release` shows users how to check them. + Automated testing -~~~~~~~~~~~~~~~~~ +================= -All pull requests and merges to 'main' branch are tested using -`Azure Pipelines `_ (configured by -``azure-pipelines.yml`` file at the root of the repository). You can find the status and results to the CI runs for your -PR on GitHub's Web UI for the pull request. You can also find links to the CI services' pages for the specific builds in -the form of "Details" links, in case the CI run fails and you wish to view the output. +All pull requests and merges to 'main' branch are tested using `GitHub Actions `_ +(configured by ``.github/workflows/check.yaml`` file at the root of the repository). You can find the status and results +to the CI runs for your PR on GitHub's Web UI for the pull request. You can also find links to the CI services' pages +for the specific builds in the form of "Details" links, in case the CI run fails and you wish to view the output. To trigger CI to run again for a pull request, you can close and open the pull request or submit another change to the pull request. If needed, project maintainers can manually trigger a restart of a job/build. +Code scanning policy +==================== + +Static analysis runs on every pull request and every push to ``main``: + +- pre-commit.ci runs ruff with every rule selected, the ``flake8-bandit`` security rules included, and `zizmor + `_ on the workflows. The ``type`` and ``type-3.9`` tox environments run ty. +- ``codeql.yaml`` runs CodeQL with the ``security-extended`` suite and the project queries under + ``.github/codeql/queries`` on the Python code, and the Actions analysis on the workflows. It also runs every Tuesday. + +A finding blocks a merge at these thresholds: + +- Any ruff, zizmor or ty finding fails a required check. +- The ``main`` ruleset blocks a merge while CodeQL reports an alert of severity ``error``, or a security alert of + severity ``high`` or ``critical``. +- Each other open CodeQL alert blocks the next release. The maintainer who cuts the release fixes it first, or dismisses + it as a false positive or as not exploitable. + +A CodeQL dismissal carries a comment that says who controls the flagged input and why it cannot cross a trust boundary +that the `threat model `_ lists. An inline +suppression names the rule it silences, and :ref:`development:Running linters` says when a ``# noqa`` fits. + NEWS entries -~~~~~~~~~~~~ +============ The ``changelog.rst`` file is managed using :pypi:`towncrier` and all non trivial changes must be accompanied by a news -entry. To add an entry to the news file, first you need to have created an issue describing the change you want to -make. A Pull Request itself *may* function as such, but it is preferred to have a dedicated issue (for example, in case -the PR ends up rejected due to code quality reasons). +entry. To add an entry to the news file, first you need to have created an issue describing the change you want to make. +A Pull Request itself *may* function as such, but it is preferred to have a dedicated issue (for example, in case the PR +ends up rejected due to code quality reasons). Once you have an issue or pull request, you take the number and you create a file inside of the ``docs/changelog`` directory named after that issue number with an extension of: @@ -158,39 +400,38 @@ directory named after that issue number with an extension of: - ``feature.rst``, - ``bugfix.rst``, - ``doc.rst``, -- ``removal.rst``, -- ``misc.rst``. +- ``deprecation.rst``, +- ``removal.rst``. Thus if your issue or PR number is ``1234`` and this change is fixing a bug, then you would create a file ``docs/changelog/1234.bugfix.rst``. PRs can span multiple categories by creating multiple files (for instance, if you added a feature and deprecated/removed the old feature at the same time, you would create -``docs/changelog/1234.bugfix.rst`` and ``docs/changelog/1234.remove.rst``). Likewise if a PR touches multiple issues/PRs -you may create a file for each of them with the same contents and :pypi:`towncrier` will deduplicate them. +``docs/changelog/1234.bugfix.rst`` and ``docs/changelog/1234.removal.rst``). Likewise if a PR touches multiple +issues/PRs you may create a file for each of them with the same contents and :pypi:`towncrier` will deduplicate them. Contents of a NEWS entry -^^^^^^^^^^^^^^^^^^^^^^^^ +------------------------ The contents of this file are reStructuredText formatted text that will be used as the content of the news file entry. -You do not need to reference the issue or PR numbers here as towncrier will automatically add a reference to all of -the affected issues when rendering the news file. +You do not need to reference the issue or PR numbers here as towncrier will automatically add a reference to all of the +affected issues when rendering the news file. In order to maintain a consistent style in the ``changelog.rst`` file, it is preferred to keep the news entry to the point, in sentence case, shorter than 120 characters and in an imperative tone -- an entry should complete the sentence ``This change will …``. In rare cases, where one line is not enough, use a summary line in an imperative tone followed -by a blank line separating it from a description of the feature/change in one or more paragraphs, each wrapped -at 120 characters. Remember that a news entry is meant for end users and should only contain details relevant to an end -user. +by a blank line separating it from a description of the feature/change in one or more paragraphs, each wrapped at 120 +characters. Remember that a news entry is meant for end users and should only contain details relevant to an end user. Choosing the type of NEWS entry -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +------------------------------- A trivial change is anything that does not warrant an entry in the news file. Some examples are: code refactors that -don't change anything as far as the public is concerned, typo fixes, white space modification, etc. To mark a PR -as trivial a contributor simply needs to add a randomly named, empty file to the ``news/`` directory with the extension -of ``.trivial``. +don't change anything as far as the public is concerned, typo fixes, white space modification, etc. To mark a PR as +trivial a contributor simply needs to add a randomly named, empty file to the ``news/`` directory with the extension of +``.trivial``. Becoming a maintainer -~~~~~~~~~~~~~~~~~~~~~ +===================== If you want to become an official maintainer, start by helping out. As a first step, we welcome you to triage issues on virtualenv's issue tracker. virtualenv maintainers provide triage abilities to contributors once they have been around @@ -200,10 +441,90 @@ initiate a vote among the existing maintainers. .. note:: - Upon becoming a maintainer, a person should be given access to various virtualenv-related tooling across - multiple platforms. These are noted here for future reference by the maintainers: + Upon becoming a maintainer, a person should be given access to various virtualenv-related tooling across multiple + platforms. These are noted here for future reference by the maintainers: - GitHub Push Access - PyPI Publishing Access - CI Administration capabilities - ReadTheDocs Administration capabilities + +Maintainer access +----------------- + +Current and backup maintainers turn on two-factor authentication for GitHub and PyPI, with a security key or passkey +where the service supports one. We review access once a year and whenever a maintainer steps back. When someone steps +back, we remove their GitHub push access, PyPI publishing access, CI administration and ReadTheDocs administration, and +rotate every secret they could read. New commit access still goes through the maintainer vote described above. + +Secrets and credentials +----------------------- + +The project stores two secrets, the client ID and the private key of the `virtualenv-release +`_ GitHub App, as ``RELEASE_APP_CLIENT_ID`` and ``RELEASE_APP_PRIVATE_KEY`` +in the ``release`` deployment environment. The push job of ``pre-release.yaml`` and the publish job of ``release.yaml`` +name that environment, and no other job can read them. Each of the two jobs trades the key for an App token that expires +within an hour and can write the contents of one repository, ``virtualenv`` or ``get-virtualenv``. + +The rest of the release needs no stored secret. PyPI accepts uploads through `trusted publishing +`_, Sigstore signs the attestations with the workflow's OIDC identity, and +the other jobs use their own ``GITHUB_TOKEN``, which the repository makes read-only unless the job asks for more. Do not +create a PyPI API token or a personal access token for the project. + +We follow these rules for the two secrets: + +- Keep them in the ``release`` environment, and out of the repository, workflow files, logs and repository-wide secrets. + Secret scanning with push protection rejects pushes that contain a known token format. +- Repository admins manage the environment secrets, and the App's owner manages its keys. +- Generate a new private key, store it in ``RELEASE_APP_PRIVATE_KEY`` and delete the old key from the App settings at + the yearly access review, when a maintainer steps back, and after a suspected leak, following the `incident response + plan `_. + +.. _current-maintainers: + +Current maintainers +------------------- + +- :user:`Bernát Gábor ` +- :user:`Rahul Devikar `, who covers when Bernát is unavailable + +Backup maintainer +----------------- + +- :user:`Paul Moore `, who steps in if both current maintainers are away + +Previous maintainers +-------------------- + +- :user:`Ian Bicking ` +- :user:`Donald Stufft ` + +********* + Roadmap +********* + +This roadmap covers October 2026 to September 2027; Bernát Gábor last updated it on 2026-09-26. It lists work that +follows from a published policy or waits on an outside decision, and it gives no release dates. We leave out work that +an open pull request delivers without waiting on anyone else. + +What we plan to do +================== + +- Add CPython 3.16 and drop CPython 3.9 under the :ref:`support policy `. CPython 3.9 + reached end of life in October 2025, so the drop can happen from May 2027; it also retires the ``_virtualenv.pth`` + distutils hook, which only 3.9 environments still receive (:issue:`3181`). +- Keep releasing on the pip and setuptools cadence that :ref:`release` describes. +- Record ``python-version`` in ``pyvenv.cfg`` (:PEP:`838`) once the PEP reaches Accepted status, restarting from the + closed :pull:`3193`. :PEP:`832` support shipped in :pull:`3204`. + +What we do not plan to do +========================= + +- Backport fixes to older release lines; see `supported versions + `_. +- Break the command line, the Python API or the plugin API, per the backwards-compatible release policy in the same + section. +- Bring back Python versions that the :ref:`support policy ` has dropped. +- Write files outside the environment it creates, such as a project-level ``.gitignore`` (:issue:`3237`). +- Install, resolve or lock project dependencies. virtualenv creates and seeds the environment; pip, uv and similar tools + manage what goes into it. diff --git a/docs/explanation.rst b/docs/explanation.rst new file mode 100644 index 000000000..4241b5025 --- /dev/null +++ b/docs/explanation.rst @@ -0,0 +1,709 @@ +############# + Explanation +############# + +This page explains the design decisions and concepts behind virtualenv. It focuses on understanding why things work the +way they do. + +************************** + virtualenv vs venv vs uv +************************** + +Since Python 3.3, the standard library includes the ``venv`` module, which provides basic virtual environment creation +following `PEP 405 `_. `uv `_ is +a newer, Rust-based tool that also creates virtual environments via ``uv venv``. + +virtualenv occupies a middle ground: faster and more featureful than ``venv``, while remaining a pure Python solution +with a plugin system for extensibility. + +.. list-table:: + :header-rows: 1 + :widths: 20 27 27 26 + + - - + - ``venv`` + - ``virtualenv`` + - `uv `_ + - - Performance + - Slowest (60s+); spawns `pip `_ as a subprocess to seed. + - Fast; caches pre-built install images, subsequent creation < 1 second. + - Fastest; Rust implementation, milliseconds. Does not seed pip/setuptools by default. + - - Extensibility + - No plugin system. + - Plugin system for discovery, creation, seeding, and activation. + - No plugin system. + - - Cross-version + - Only the Python version it runs under. + - Any installed Python via auto-discovery (registry, uv-managed, PATH). + - Any installed or uv-managed Python. + - - Upgradeability + - Tied to Python releases. + - Independent via `PyPI `_. + - Independent via its own release cycle. + - - Programmatic API + - Basic ``create()`` function only. + - Full Python API; can describe environments without creating them. Used by `tox `_, + `poetry `_, `pipx `_, etc. + - Command line only. + - - Type annotations + - No ``py.typed`` marker; limited annotations. + - Fully typed with :PEP:`561` ``py.typed`` marker; checked by `ty `_. + - Not applicable (Rust binary). + - - Best for + - Zero dependencies, basic needs. + - Plugin extensibility, programmatic API, tool compatibility (`tox `_, + `virtualenvwrapper `_). + - Maximum speed, already using ``uv`` for package management. + +.. mermaid:: + + flowchart TD + A{Need plugins or programmatic API?} -->|Yes| V[virtualenv] + A -->|No| B{Already using uv?} + B -->|Yes| U[uv venv] + B -->|No| C{Can install external tools?} + C -->|Yes| D{Speed matters?} + C -->|No| VENV[venv] + D -->|Yes| U + D -->|No| V + + style A fill:#d97706,stroke:#b45309,color:#fff + style B fill:#d97706,stroke:#b45309,color:#fff + style C fill:#d97706,stroke:#b45309,color:#fff + style D fill:#d97706,stroke:#b45309,color:#fff + style V fill:#16a34a,stroke:#15803d,color:#fff + style U fill:#2563eb,stroke:#1d4ed8,color:#fff + style VENV fill:#7c3aed,stroke:#6d28d9,color:#fff + +*************************** + Supported Python versions +*************************** + +virtualenv distinguishes between the **host** interpreter (the Python running virtualenv itself) and the **target** +interpreter (the Python for which the virtual environment is created). When ``--python`` is not specified, the host and +target are the same interpreter. + +Host interpreter +================ + +virtualenv requires CPython or PyPy **3.9 or later** as the host interpreter. This is enforced by ``requires-python >= +3.9`` in the package metadata. + +Target interpreter +================== + +The target interpreter can differ from the host. virtualenv can create virtual environments for any Python interpreter +it can discover on the system, provided a matching creator exists. + +.. list-table:: + :header-rows: 1 + :widths: 20 15 65 + + - - Implementation + - Platforms + - Notes + - - CPython + - Linux, macOS, Windows + - 3.9+. Free-threaded builds supported on 3.13+. Includes macOS framework, Homebrew, Microsoft Store, and Windows + debug build support. + - - PyPy + - Linux, macOS, Windows + - 3.9+. + - - GraalPy + - Linux, macOS, Windows + - 24.1+. Minimal test coverage, marked experimental. + - - RustPython + - Linux, macOS, Windows + - Minimal test coverage, marked experimental. + +Seed packages (``pip``, ``setuptools``) are bundled for CPython 3.9 through 3.16. A target newer than the highest +bundled version reuses the newest bundle. A target older than the oldest bundled version has no compatible bundled +wheel, so the bundled seeders refuse it before the environment is created; pass ``--no-seed`` for an empty environment, +or select a seeder that ships wheels for that version. + +********************** + How virtualenv works +********************** + +Python packaging often faces a fundamental problem: different applications require different versions of the same +library. If Application A needs ``requests==2.25.1`` but Application B needs ``requests==2.28.0``, installing both into +the global site-packages directory creates a conflict. Only one version can exist in a given location. + +virtualenv solves this by creating isolated Python environments. Each environment has its own installation directories +and can maintain its own set of installed packages, independent of other environments and the system Python. + +virtualenv operates in two distinct phases: + +.. mermaid:: + + flowchart TD + Start([virtualenv command]) --> Phase1[Phase 1: Python Discovery] + Phase1 --> Discover{Find Python interpreter} + Discover -->|Default| SameVersion[Use virtualenv's own Python] + Discover -->|--python flag| CustomVersion[Use specified Python] + CustomVersion --> Phase2[Phase 2: Environment Creation] + SameVersion --> Phase2 + Phase2 --> CreatePython[Create Python matching target interpreter] + CreatePython --> SeedPackages[Install seed packages: pip, setuptools, wheel] + SeedPackages --> ActivationScripts[Install activation scripts] + ActivationScripts --> VCSIgnore[Create VCS ignore files] + VCSIgnore --> Record[Point .venv redirect at it] + Record --> Complete([Virtual environment ready]) + + style Start fill:#2563eb,stroke:#1d4ed8,color:#fff + style Phase1 fill:#6366f1,stroke:#4f46e5,color:#fff + style Phase2 fill:#6366f1,stroke:#4f46e5,color:#fff + style Complete fill:#16a34a,stroke:#15803d,color:#fff + style Discover fill:#d97706,stroke:#b45309,color:#fff + +**Phase 1: Discover a Python interpreter** + virtualenv first identifies which Python interpreter to use as the template for the virtual environment. By default, + it uses the same Python version that virtualenv itself is running on. You can override this with the ``--python`` + flag to specify a different interpreter. + +**Phase 2: Create the virtual environment** + Once the target interpreter is identified, virtualenv creates the environment in five steps: + + 1. Create a Python executable matching the target interpreter + 2. Install seed packages (pip, setuptools, wheel) to enable package installation + 3. Install activation scripts for various shells + 4. Create VCS ignore files (currently Git's ``.gitignore``, skip with ``--no-vcs-ignore``) + 5. In a project folder, point a ``.venv`` redirect file beside the environment at it so editors and type checkers + can find it (`PEP 832 `_, provisional; force with ``--venv-redirect``, skip + with ``--no-venv-redirect``) + +An important design principle: virtual environments are not self-contained. A complete Python installation consists of +thousands of files, and copying all of them into every virtual environment would be wasteful. Instead, virtual +environments are lightweight shells that borrow most content from the system Python. They contain only what's needed to +redirect Python's behavior. + +``pyvenv.cfg`` is what makes that redirection work. The interpreter reads it at startup to find its base installation, +so the file is the environment rather than a description of it. That also makes it the cheapest place for other tools to +learn about the environment, which is why it accumulated three keys carrying the same version at different precisions. +`PEP 838 `_ settles which one to read by requiring ``python-version``, holding the +feature release and nothing more, since a wheel tag and a type-checker target both stop at the minor version. +``version`` and ``version_info`` stay for the callers that need a patch level or a release level, and +:doc:`reference/files` sets out the difference. + +This design has two implications: + +- Environment creation is fast because only a small number of files need to be created. +- Upgrading the system Python might affect existing virtual environments, since they reference the system Python's + standard library and binary extensions. + +The Python executable in a virtual environment is effectively isolated from the one used to create it, but the +supporting files are shared. + +.. warning:: + + If you upgrade your system Python, existing virtual environments will still report the old version (the version + number is embedded in the Python executable itself), but they will use the new version's standard library and binary + extensions. This normally works without issues, but be aware that the environment is effectively running a hybrid of + old and new Python versions. + +****************** + Python discovery +****************** + +Before creating a virtual environment, virtualenv must locate a Python interpreter. The interpreter determines the +virtual environment's Python version, implementation (CPython, PyPy, etc.), and architecture (32-bit or 64-bit). + +The ``--python`` flag accepts several specifier formats: + +**Path specifier** + An absolute or relative path to a Python executable, such as ``/usr/bin/python3.14`` or ``./python``. + +**Version specifier** + A string following the format ``{implementation}{version}{architecture}{machine}`` where: + + - Implementation is alphabetic characters (``python`` means any implementation; if omitted, defaults to ``python``). + - Version is dot-separated numbers, optionally followed by ``t`` for free-threading builds. + - Architecture is ``-64`` or ``-32`` (if omitted, means any architecture). + - Machine is the CPU instruction set architecture, e.g. ``-arm64``, ``-x86_64``, ``-aarch64`` (if omitted, means any + machine). Cross-platform aliases are normalized automatically (``amd64`` ↔ ``x86_64``, ``aarch64`` ↔ ``arm64``). + + Examples: + + - ``python3.14.1`` - Any Python implementation with version 3.14.1 + - ``3`` - Any Python implementation with major version 3 + - ``3.13t`` - Any Python implementation version 3.13 with free-threading enabled + - ``cpython3`` - CPython implementation with major version 3 + - ``pypy2`` - PyPy implementation with major version 2 + - ``cpython3.12-64-arm64`` - CPython 3.12, 64-bit, ARM64 architecture + - ``3.11-64-x86_64`` - Any implementation, version 3.11, 64-bit, x86_64 architecture + - ``rustpython`` - RustPython implementation + +**PEP 440 version specifier** + Version constraints using PEP 440 operators: + + - ``>=3.12`` - Any Python 3.12 or later + - ``~=3.11.0`` - Compatible with Python 3.11.0 + - ``cpython>=3.10`` - CPython 3.10 or later + +When you provide a specifier, virtualenv searches for matching interpreters using this strategy: + +.. mermaid:: + + flowchart TD + Start([Python specifier provided]) --> Windows{Running on Windows?} + Windows -->|Yes| Registry[Check Windows Registry per PEP-514] + Windows -->|No| UVManaged + Registry --> RegistryMatch{Match found?} + RegistryMatch -->|Yes| Found([Use matched Python]) + RegistryMatch -->|No| UVManaged[Check uv-managed Python installations] + UVManaged --> UVMatch{Match found?} + UVMatch -->|Yes| Found + UVMatch -->|No| PATH[Search PATH for matching executable] + PATH --> PATHMatch{Match found?} + PATHMatch -->|Yes| Found + PATHMatch -->|No| NotFound([Discovery fails]) + + style Start fill:#2563eb,stroke:#1d4ed8,color:#fff + style Found fill:#16a34a,stroke:#15803d,color:#fff + style NotFound fill:#dc2626,stroke:#b91c1c,color:#fff + style Windows fill:#d97706,stroke:#b45309,color:#fff + style RegistryMatch fill:#d97706,stroke:#b45309,color:#fff + style UVMatch fill:#d97706,stroke:#b45309,color:#fff + style PATHMatch fill:#d97706,stroke:#b45309,color:#fff + +1. **Windows Registry** (Windows only): Check registered Python installations per `PEP 514 + `_. +2. **uv-managed installations**: Check the ``UV_PYTHON_INSTALL_DIR`` environment variable or platform-specific uv Python + directories for managed Python installations. +3. **PATH search**: Search for executables on the ``PATH`` environment variable with names matching the specification. + +Version manager shim resolution +=============================== + +Version managers like `pyenv `_, `mise `_, and `asdf +`_ place lightweight shim scripts on ``PATH`` that delegate to the real Python binary. When +virtualenv discovers a Python interpreter by running it as a subprocess, shims may resolve to the wrong Python version +(typically the system Python) because the shim's resolution logic depends on shell environment state that doesn't fully +propagate to child processes. + +virtualenv detects shims by checking whether the candidate executable lives in a known shim directory +(``$PYENV_ROOT/shims``, ``$MISE_DATA_DIR/shims``, or ``$ASDF_DATA_DIR/shims``). When a shim is detected, virtualenv +bypasses it and locates the real binary directly under the version manager's ``versions`` directory, using the active +version from: + +1. The ``PYENV_VERSION`` environment variable (colon-separated for multiple versions). +2. A ``.python-version`` file in the current directory or any parent directory. +3. The global version file at ``$PYENV_ROOT/version``. + +This convention is shared across pyenv, mise, and asdf, so the same resolution logic works for all three. + +.. warning:: + + Virtual environments typically reference the system Python's standard library. If you upgrade the system Python, the + virtual environment will report the old version (embedded in its Python executable) but will actually use the new + version's standard library content. This can cause confusion when debugging version-specific behavior. + + If you use a virtual environment's Python as the target for creating another virtual environment, virtualenv will + detect the system Python version and create an environment matching the actual (upgraded) version, not the version + reported by the virtual environment. + +********** + Creators +********** + +Creators are responsible for constructing the virtual environment structure. virtualenv supports two types of creators: + +**venv creator** + This creator delegates the entire creation process to the standard library's ``venv`` module, following `PEP 405 + `_. The venv creator requires spawning a subprocess to invoke the venv + module, unless virtualenv is installed in the system Python. + + The subprocess overhead can be significant, especially on Windows where process creation is expensive. + +**builtin creator** + This creator means virtualenv performs the creation itself by knowing exactly which files to create and which system + files to reference. The builtin creator is actually a family of specialized creators for different combinations of + Python implementation (CPython, PyPy, GraalPy, RustPython) and platform (Windows, POSIX). The name ``builtin`` is an + alias that selects the first available builtin creator for the target environment. + + Because builtin creators don't require subprocess invocation, they're generally faster than the venv creator. + +.. mermaid:: + + flowchart TD + Start([Select creator]) --> Builtin{Builtin creator available?} + Builtin -->|Yes| UseBuiltin([Use builtin creator - faster, no subprocess]) + Builtin -->|No| UseVenv([Use venv creator - delegates to stdlib]) + + style Start fill:#2563eb,stroke:#1d4ed8,color:#fff + style UseBuiltin fill:#16a34a,stroke:#15803d,color:#fff + style UseVenv fill:#7c3aed,stroke:#6d28d9,color:#fff + style Builtin fill:#d97706,stroke:#b45309,color:#fff + +virtualenv defaults to using the builtin creator if one is available for the target environment, falling back to the +venv creator otherwise. + +Whichever creator runs, the interpreter ends up reachable under several names, listed in +:doc:`reference/environment-layout`. Tools reach for different ones - a shebang may say ``python``, a CI job may call +the versioned name - and an environment offering only one of them would break the others. The builtin creators take the +version-bearing names from the target interpreter and add the host executable's own file name, so a host called +``pypy3.11`` or ``python_d.exe`` stays reachable under the name its callers already use. + +Windows copies rather than symlinks, because symlinking the interpreter there is unreliable (`bpo-42013 +`_). From CPython 3.13 the file copied is the ``venvlauncher.exe`` redirector out of +the host's standard library rather than the interpreter itself. The redirector reads ``pyvenv.cfg`` and hands off to the +base prefix, which saves copying the DLLs and the standard library zip into every environment. Which name a copy carries +makes no difference to where it lands, since the redirector resolves its target from a value compiled into it rather +than from its own file name. + +********* + Seeders +********* + +After creating the virtual environment structure, virtualenv installs seed packages that enable package management +within the environment. The seed packages are: + +- ``pip`` - The package installer for Python (always installed). +- ``setuptools`` - Package development and installation library (disabled by default on Python 3.12+). + +virtualenv supports two seeding methods with dramatically different performance characteristics: + +**pip seeder** + This method uses the bundled pip wheel to install seed packages by spawning a child pip process. The subprocess + performs a full installation, including unpacking wheels and generating metadata. This method is reliable but slow, + typically consuming 98% of the total virtual environment creation time. + +**app-data seeder** + This method creates reusable install images in a user application data directory. The first time you create an + environment with specific seed package versions, the app-data seeder builds complete install images and stores them + in the cache. Subsequent environment creations simply link or copy these pre-built images into the virtual + environment's ``site-packages`` directory. + + Performance comparison for creating virtual environments: + + .. mermaid:: + + xychart-beta horizontal + title "Seeding time (seconds, lower is better)" + x-axis ["pip seeder (70s)", "app-data copy Win (8s)", "app-data symlink Win (0.8s)", "app-data symlink Linux/macOS (0.1s)"] + y-axis "Seconds" 0 --> 80 + bar [70, 8, 0.8, 0.1] + + On platforms that support symlinks efficiently (Linux, macOS), the app-data seeder provides nearly instant seeding. + + You can override the cache location using the ``VIRTUALENV_OVERRIDE_APP_DATA`` environment variable. + +.. _wheels: + +Wheel acquisition +================= + +Both seeding methods require wheel files for the seed packages. virtualenv acquires wheels using a priority system: + +.. mermaid:: + + flowchart TD + Start([Need wheel file]) --> Embedded{Found in embedded wheels?} + Embedded -->|Yes| UseEmbedded([Use embedded wheel]) + Embedded -->|No| Upgraded{Found in upgraded wheels?} + Upgraded -->|Yes| UseUpgraded([Use upgraded wheel]) + Upgraded -->|No| Extra{Found in extra-search-dir?} + Extra -->|Yes| UseExtra([Use extra wheel]) + Extra -->|No| Download{Download enabled?} + Download -->|Yes| DownloadPyPI([Download from PyPI]) + Download -->|No| Fail([Seeding fails]) + + style Start fill:#2563eb,stroke:#1d4ed8,color:#fff + style UseEmbedded fill:#16a34a,stroke:#15803d,color:#fff + style UseUpgraded fill:#16a34a,stroke:#15803d,color:#fff + style UseExtra fill:#16a34a,stroke:#15803d,color:#fff + style DownloadPyPI fill:#16a34a,stroke:#15803d,color:#fff + style Fail fill:#dc2626,stroke:#b91c1c,color:#fff + style Embedded fill:#d97706,stroke:#b45309,color:#fff + style Upgraded fill:#d97706,stroke:#b45309,color:#fff + style Extra fill:#d97706,stroke:#b45309,color:#fff + style Download fill:#d97706,stroke:#b45309,color:#fff + +**Embedded wheels** + virtualenv ships with a set of wheels bundled directly into the package. These are tested with the virtualenv + release and provide a baseline set of seed packages. Different Python versions require different package versions, + so virtualenv bundles multiple wheels to support its wide Python version range. + +**Upgraded embedded wheels** + Users can manually upgrade the embedded wheels by running virtualenv with the ``--upgrade-embed-wheels`` flag. This + fetches newer versions of seed packages from PyPI and stores them in the user application data directory. Subsequent + virtualenv invocations will use these upgraded wheels instead of the embedded ones. + + virtualenv can also perform periodic automatic upgrades (see below). + +**Extra search directories** + Users can specify additional directories containing wheels using the ``--extra-search-dir`` flag. This is useful in + air-gapped environments or when using custom package builds. + +**PyPI download** + If no suitable wheel is found in the above locations, or if the ``--download`` flag is set, virtualenv will use pip + to download the latest compatible version from PyPI. + +Periodic update mechanism +========================= + +To keep the seed packages reasonably current without requiring users to manually upgrade virtualenv or run +``--upgrade-embed-wheels``, virtualenv implements a periodic automatic update system: + +.. mermaid:: + + timeline + title Periodic update safety gates + section PyPI release + Package published : New wheel available on PyPI + section 28-day wait + Day 1-28 : Wheel is too new, ignored by virtualenv + section Check interval + Every 14 days : virtualenv checks for eligible wheels + section 1-hour hold + After download : Wheel downloaded but not yet used + +1 hour : Wheel becomes active for new environments + +The 28-day waiting period protects users from automatically adopting newly released packages that might contain bugs. +The 1-hour delay after download ensures continuous integration systems don't start using different package versions +mid-run, which could cause confusing test failures. + +You can disable the periodic update mechanism with the ``--no-periodic-update`` flag. + +.. _distribution_wheels: + +Distribution maintainer patching +================================ + +Operating system distributions and package managers sometimes need to customize which seed package versions virtualenv +uses. They want to align virtualenv's bundled packages with system package versions. + +Distributions can patch the ``virtualenv.seed.wheels.embed`` module, replacing the ``get_embed_wheel`` function with +their own implementation that returns distribution-provided wheels. If they want to use virtualenv's test suite for +validation, they should also provide the ``BUNDLE_FOLDER``, ``BUNDLE_SUPPORT``, ``MIN``, ``MAX``, and +``OLDEST_SUPPORTED`` variables. ``OLDEST_SUPPORTED`` (the parsed form of ``MIN``) sets the floor below which the bundled +seeders refuse to seed, so a distribution that bundles wheels for an older Python must lower it to match. + +Distributions should also consider patching ``virtualenv.seed.embed.base_embed.PERIODIC_UPDATE_ON_BY_DEFAULT`` to +``False``, allowing the system package manager to control seed package updates rather than virtualenv's periodic update +mechanism. Users can still manually request upgrades via ``--upgrade-embed-wheels``, but automatic updates won't +interfere with system-managed packages. + +************ + Activators +************ + +Activation scripts modify the current shell environment to prioritize the virtual environment's executables. This is +purely a convenience mechanism - you can always use absolute paths to virtual environment executables without +activating. + +What activation does: + +.. mermaid:: + + flowchart TD + Before([Before activation]) --> ModifyPATH[Prepend venv/bin to PATH] + ModifyPATH --> SetVENV[Set VIRTUAL_ENV variable] + SetVENV --> SetPROMPT[Set VIRTUAL_ENV_PROMPT variable] + SetPROMPT --> SetPKG[Set PKG_CONFIG_PATH] + SetPKG --> ModifyPrompt[Modify shell prompt] + ModifyPrompt --> After([After activation]) + + style Before fill:#2563eb,stroke:#1d4ed8,color:#fff + style After fill:#16a34a,stroke:#15803d,color:#fff + style ModifyPATH fill:#6366f1,stroke:#4f46e5,color:#fff + style SetVENV fill:#6366f1,stroke:#4f46e5,color:#fff + style SetPROMPT fill:#6366f1,stroke:#4f46e5,color:#fff + style SetPKG fill:#6366f1,stroke:#4f46e5,color:#fff + style ModifyPrompt fill:#6366f1,stroke:#4f46e5,color:#fff + +**PATH modification** + The activation script prepends the virtual environment's ``bin`` directory (``Scripts`` on Windows) to the ``PATH`` + environment variable. This ensures that when you run ``python``, ``pip``, or other executables, the shell finds the + virtual environment's versions first. + +**Environment variables** + Activation sets several environment variables: + + - ``VIRTUAL_ENV`` - Absolute path to the virtual environment directory. + - ``VIRTUAL_ENV_PROMPT`` - The prompt prefix (the environment name or custom value from ``--prompt``). + - ``PKG_CONFIG_PATH`` - Modified to include the virtual environment's ``lib/pkgconfig`` directory. + +**Prompt modification** + By default, activation prepends the environment name to your shell prompt, typically shown as ``(venv)`` before the + regular prompt. This visual indicator helps you remember which environment is active. You can customize this with + the ``--prompt`` flag when creating the environment, or disable it entirely by setting the + ``VIRTUAL_ENV_DISABLE_PROMPT`` environment variable. + +**Deactivation** + Activation scripts also provide a ``deactivate`` command that reverses the changes, restoring your original PATH and + removing the environment variables and prompt modifications. + +virtualenv provides activation scripts for multiple shells: + +- `Bash `_ (``activate``) +- `Fish `_ (``activate.fish``) +- `Csh/Tcsh `_ (``activate.csh``) +- `PowerShell `_ (``activate.ps1``) +- `Windows Batch `_ + (``activate.bat``) +- `Nushell `_ (``activate.nu``) +- Python (``activate_this.py``) -- for programmatic activation from within a running Python process, see + :ref:`how-to/usage:Programmatic activation` + +.. note:: + + On Windows 7 and later, PowerShell's default execution policy is ``Restricted``, which prevents running the + ``activate.ps1`` script. You can allow locally-generated scripts to run by changing the execution policy: + + .. code-block:: powershell + + Set-ExecutionPolicy RemoteSigned + + Since virtualenv generates ``activate.ps1`` locally for each environment, PowerShell considers it a local script + rather than a remote one and allows execution under the ``RemoteSigned`` policy. + +Remember: activation is optional. The following commands are equivalent: + +.. code-block:: console + + # With activation + source venv/bin/activate + python script.py + deactivate + + # Without activation + venv/bin/python script.py + +For a deeper dive into how activation works under the hood, see Allison Kaptur's blog post `There's no magic: virtualenv +edition `_, which explains how virtualenv uses +``PATH`` and ``PYTHONHOME`` to isolate virtual environments. + +.. _release-integrity: + +******************* + Release integrity +******************* + +virtualenv installs the ``pip`` and ``setuptools`` wheels it bundles into every environment it creates, so a tampered +release would reach all of them. Each release lets you check where its files came from without trusting the maintainers' +machines. :doc:`how-to/verify-release` walks through the checks and :doc:`reference/release-artifacts` lists the files +involved. + +**Trusted publishing** + The release workflow holds no PyPI API token. Its publish job exchanges a short-lived GitHub OIDC token for upload + rights through PyPI's `trusted publishing `_, and the publish attestation + PyPI keeps for each file names that publisher: the ``release.yaml`` workflow of ``pypa/virtualenv`` in the + ``release`` environment. + +**Provenance** + For each file, the workflow signs a statement that binds the file's SHA-256 to the workflow identity: repository, + workflow file, tag and commit. `Sigstore `_ issues the signing certificate and records it + in a public transparency log. PyPI stores the statements for the wheel and sdist; GitHub stores the one for the + zipapp and the SBOM attestations. Verifying a file recomputes its digest and checks it against a signed statement + whose identity you name. + +**Immutable releases** + From 21.11.0 on, once the workflow publishes a GitHub release, GitHub refuses changes to its tag and assets and + signs a release attestation over the tag's commit and each asset's SHA-256. A stolen maintainer token cannot swap + the zipapp or an SBOM on a published release. The release attestation names no workflow, so pair it with the + provenance check. + +**Reproducible builds** + Provenance tells you which workflow run built a file, but you still trust that run. The release pins every timestamp + to ``SOURCE_DATE_EPOCH``, the commit time of the tag, so you can rebuild the sdist, the wheel and the zipapp from + the tag yourself and compare the bytes. The SBOMs record the Python version and build tools the release used, which + a rebuild must match, and a lock file pins the distributions the zipapp bundles. The SBOMs leave out the machine + that ran the build, which the provenance attestation already names. + +**The SBOMs** + Dependency scanners find the packages a project declares, and virtualenv declares neither ``pip`` nor + ``setuptools``: it ships them as data files. The CycloneDX SBOM in the wheel names each bundled wheel with its hash, + license and vendored packages, so a scanner can match them against vulnerability databases. The release also + publishes it as SPDX for tools that read only that format. The zipapp gets its own SBOM because it bundles more: the + runtime dependencies for every supported Python, each in the version that Python loads. + +What a successful check proves: + +- The file is byte-identical to one that ``release.yaml`` in ``pypa/virtualenv`` built and signed. +- With ``--source-ref`` or ``--signer-workflow``, the run came from that tag and workflow file. + +What it does not prove: + +- That the source at that tag is free of bugs or malicious code. Provenance ties a file to a commit; review of the + commit is a separate matter. +- That the actions and build dependencies the workflow pulled in were not compromised. The SBOM records the build + environment so you can audit it after the fact. +- That the bundled ``pip`` and ``setuptools`` have no known vulnerabilities. The SBOM lets you look them up. +- Anything about wheels virtualenv fetches at run time. ``--download``, ``--upgrade-embed-wheels`` and the periodic + update take ``pip`` and ``setuptools`` from PyPI, outside virtualenv's attestations. +- That you have the latest release. The bootstrap zipapp in particular can trail it. + +*********************** + Environment discovery +*********************** + +Editors, type checkers and task runners need to find a project's environment, and they cannot count on an activated +shell to tell them where it is. Launching an editor on a fresh checkout leaves it guessing. Each tool has answered that +by hard-coding a search per environment manager it wants to support, which is why editor support for any new tool lags +behind the tool itself. + +`PEP 832 `_ gives the guess a fixed place to look: ``.venv`` in the project root, +either the environment itself or a redirect file holding a single line with the path of one. virtualenv creates +environments, so it writes the redirect; reading it belongs to the tools consuming it. + +virtualenv is a low-level tool, and most environments it builds are not a project's default, such as scratch +environments or the ones tox and nox create for their own use. By default it writes the redirect when the environment +lands in a project folder, one holding a ``pyproject.toml``, and that folder has no ``.venv`` yet: + +.. code-block:: console + + $ ls + pyproject.toml src + $ virtualenv env + $ virtualenv scratch + $ cat .venv + env + +**The first environment in a project claims the default** + Later environments leave the redirect alone, so a throwaway environment next to your working one does not move your + editor to it. Pass ``--venv-redirect`` to point the redirect at the new environment. + +**Outside a project folder, virtualenv writes nothing** + ``virtualenv foo`` in a folder without a ``pyproject.toml`` leaves the folder as it was. ``--venv-redirect`` writes + the redirect there anyway. + +**A ``.venv`` folder stays** + Creating ``.venv`` itself writes nothing, and where a ``.venv`` folder or symlink exists virtualenv leaves it alone, + since it already is the default. + +**Other tools' redirects stay** + Even with ``--venv-redirect``, virtualenv honors the PEP's request not to overwrite a redirect another tool wrote. + The file carries no marker of who wrote it, so virtualenv judges by the target: it takes over the redirect only when + the ``pyvenv.cfg`` there carries the ``virtualenv`` key. A redirect written by uv, ``venv`` or you stays, and so + does one whose target is gone, since it may still resolve inside a container or on another machine. + +**The command line wins** + ``--venv-redirect`` and ``--no-venv-redirect`` set one choice, and a flag on the command line beats the environment + variable and the configuration file, so a single run can override a standing setting in either direction. + +**A failed write keeps the environment** + A write failure logs a warning and leaves you with a working environment. Discovery is a convenience for other + tools, so it does not get to fail the job you asked for. + +Why not a list +============== + +An earlier draft of the PEP listed every environment in a ``.python-envs`` file. The PEP went back to a single redirect +after workflow tool authors warned that tools sharing one list would overwrite each other's entries and change the +default under the user. virtualenv follows the PEP, so it writes one line. + +Provisional status +================== + +The feature is provisional. PEP 832 is still a draft whose format changed twice during review, and virtualenv tracks the +PEP rather than its own earlier behavior, so a minor or patch release may change what virtualenv writes in backward +incompatible ways. Pass ``--no-venv-redirect`` if you need a stable project folder until the Steering Council accepts +the PEP. + +********** + See also +********** + +- :doc:`how-to/usage` - Practical guides for common virtualenv tasks. +- :doc:`reference/cli` - Complete CLI reference documentation. diff --git a/docs/extend.rst b/docs/extend.rst deleted file mode 100644 index 6dc107e3b..000000000 --- a/docs/extend.rst +++ /dev/null @@ -1,88 +0,0 @@ -Extend functionality -==================== - -``virtualenv`` allows one to extend the builtin functionality via a plugin system. To add a plugin you need to: - -- write a python file containing the plugin code which follows our expected interface, -- package it as a python library, -- install it alongside the virtual environment. - -Python discovery ----------------- - -The python discovery mechanism is a component that needs to answer the following question: based on some type of user -input give me a Python interpreter on the machine that matches that. The builtin interpreter tries to discover -an installed Python interpreter (based on PEP-515 and ``PATH`` discovery) on the users machine where the user input is a -python specification. An alternative such discovery mechanism for example would be to use the popular -`pyenv `_ project to discover, and if not present install the requested Python -interpreter. Python discovery mechanisms must be registered under key ``virtualenv.discovery``, and the plugin must -implement :class:`virtualenv.discovery.discover.Discover`: - -.. code-block:: ini - - virtualenv.discovery = - pyenv = virtualenv_pyenv.discovery:PyEnvDiscovery - - -.. currentmodule:: virtualenv.discovery.discover - -.. autoclass:: Discover - :undoc-members: - :members: - - -Creators --------- -Creators are what actually perform the creation of a virtual environment. The builtin virtual environment creators -all achieve this by referencing a global install; but would be just as valid for a creator to install a brand new -entire python under the target path; or one could add additional creators that can create virtual environments for other -python implementations, such as IronPython. They must be registered under and entry point with key -``virtualenv.create`` , and the class must implement :class:`virtualenv.create.creator.Creator`: - -.. code-block:: ini - - virtualenv.create = - cpython3-posix = virtualenv.create.via_global_ref.builtin.cpython.cpython3:CPython3Posix - -.. currentmodule:: virtualenv.create.creator - -.. autoclass:: Creator - :undoc-members: - :members: - :exclude-members: run, set_pyenv_cfg, debug_script, debug_script, validate_dest, debug - - -Seed mechanism --------------- - -Seeders are what given a virtual environment will install somehow some seed packages into it. They must be registered -under and entry point with key ``virtualenv.seed`` , and the class must implement -:class:`virtualenv.seed.seeder.Seeder`: - -.. code-block:: ini - - virtualenv.seed = - db = virtualenv.seed.fromDb:InstallFromDb - -.. currentmodule:: virtualenv.seed.seeder - -.. autoclass:: Seeder - :undoc-members: - :members: - -Activation scripts ------------------- -If you want add an activator for a new shell you can do this by implementing a new activator. They must be registered -under an entry point with key ``virtualenv.activate`` , and the class must implement -:class:`virtualenv.activation.activator.Activator`: - -.. code-block:: ini - - virtualenv.activate = - bash = virtualenv.activation.bash:BashActivator - -.. currentmodule:: virtualenv.activation.activator - -.. autoclass:: Activator - :undoc-members: - :members: diff --git a/docs/how-to/install.rst b/docs/how-to/install.rst new file mode 100644 index 000000000..ecb5ff8b8 --- /dev/null +++ b/docs/how-to/install.rst @@ -0,0 +1,105 @@ +#################### + Install virtualenv +#################### + +virtualenv is a command-line tool, so it should be installed in an isolated environment rather than into your system +Python. Pick the method that fits your setup: + +- `uv `_ -- fast, modern Python package manager. Use this if you already have ``uv`` or are + starting fresh. +- `pipx `_ -- installs Python CLI tools in isolated environments. Use this if you already + have ``pipx`` set up. +- `pip `_ -- the standard Python package installer. Use ``--user`` to avoid modifying + system packages. May not work on distributions with externally-managed Python environments. +- `zipapp `_ -- a self-contained executable requiring no installation. + Use this in CI or environments where you cannot install packages. + +.. mermaid:: + + flowchart TD + A{Can you install packages?} -->|Yes| B{Have uv?} + A -->|No| Z[zipapp] + B -->|Yes| U[uv tool install] + B -->|No| C{Have pipx?} + C -->|Yes| P[pipx install] + C -->|No| D[pip install --user] + + style A fill:#d97706,stroke:#b45309,color:#fff + style B fill:#d97706,stroke:#b45309,color:#fff + style C fill:#d97706,stroke:#b45309,color:#fff + style U fill:#16a34a,stroke:#15803d,color:#fff + style P fill:#16a34a,stroke:#15803d,color:#fff + style D fill:#7c3aed,stroke:#6d28d9,color:#fff + style Z fill:#7c3aed,stroke:#6d28d9,color:#fff + +.. tab:: uv + + Install virtualenv as a `uv tool `_: + + .. code-block:: console + + $ uv tool install virtualenv + + Install the development version: + + .. code-block:: console + + $ uv tool install git+https://github.com/pypa/virtualenv.git@main + +.. tab:: pipx + + Install virtualenv using `pipx `_: + + .. code-block:: console + + $ pipx install virtualenv + + Install the development version: + + .. code-block:: console + + $ pipx install git+https://github.com/pypa/virtualenv.git@main + +.. tab:: pip + + Install virtualenv using `pip `_: + + .. code-block:: console + + $ python -m pip install --user virtualenv + + Install the development version: + + .. code-block:: console + + $ python -m pip install git+https://github.com/pypa/virtualenv.git@main + + .. warning:: + + Some Linux distributions use system-managed Python environments. If you encounter errors about externally-managed + environments, use ``uv tool`` or ``pipx`` instead. + +.. tab:: zipapp + + Download the zipapp file and run it directly: + + .. code-block:: console + + $ python virtualenv.pyz --help + + Download the latest version from https://bootstrap.pypa.io/virtualenv.pyz or a specific version from + ``https://bootstrap.pypa.io/virtualenv/x.y/virtualenv.pyz``. + + Each GitHub release also attaches the zipapp with a provenance bundle; :doc:`verify-release` shows how to check it. + +********************* + Verify installation +********************* + +Check the installed version: + +.. code-block:: console + + $ virtualenv --version + +See :doc:`../reference/compatibility` for supported Python versions. diff --git a/docs/how-to/usage.rst b/docs/how-to/usage.rst new file mode 100644 index 000000000..8dcef443f --- /dev/null +++ b/docs/how-to/usage.rst @@ -0,0 +1,451 @@ +################ + Use virtualenv +################ + +************************* + Select a Python version +************************* + +By default, virtualenv uses the same Python version it runs under. Override this with ``--python`` or ``-p``. + +Using version specifiers +======================== + +Specify a Python version by name or version number: + +.. code-block:: console + + $ virtualenv -p python3.14 venv + $ virtualenv -p 3.10 venv + $ virtualenv -p pypy3 venv + $ virtualenv -p rustpython venv + +Using PEP 440 specifiers +======================== + +Use `PEP 440 `_ version specifiers to match Python versions: + +.. code-block:: console + + $ virtualenv --python ">=3.12" venv + $ virtualenv --python "~=3.11.0" venv + $ virtualenv --python "cpython>=3.10" venv + +- ``>=3.12`` -- any Python 3.12 or later. +- ``~=3.11.0`` -- compatible release, equivalent to ``>=3.11.0, <3.12.0`` (any 3.11.x patch). +- ``cpython>=3.10`` -- restrict to CPython implementation, 3.10 or later. + +Using free-threading Python +=========================== + +Create an environment with `free-threading Python `_: + +.. code-block:: console + + $ virtualenv -p 3.13t venv + +Targeting a specific CPU architecture +===================================== + +On machines that support multiple architectures — such as Apple Silicon (arm64 + x86_64 via Rosetta) or Windows on ARM — +you can request a specific CPU architecture by appending it to the spec string: + +.. code-block:: console + + $ virtualenv -p cpython3.12-64-arm64 venv + $ virtualenv -p 3.11-64-x86_64 venv + +Cross-platform aliases are normalized automatically, so ``amd64`` and ``x86_64`` are treated as equivalent, as are +``aarch64`` and ``arm64``. If omitted, any architecture matches (preserving existing behavior). + +Using absolute paths +==================== + +Specify the full path to a Python interpreter: + +.. code-block:: console + + $ virtualenv -p /usr/bin/python3.9 venv + +Using ``--try-first-with`` +========================== + +Use ``--try-first-with`` to provide a hint about which Python to check first. Unlike ``--python``, this is a hint rather +than a rule. The interpreter at this path is checked first, but only used if it matches the ``--python`` constraint. + +.. code-block:: console + + $ virtualenv --python ">=3.10" --try-first-with /usr/bin/python3.9 venv + +In this example, /usr/bin/python3.9 is checked first but rejected because it does not satisfy the >=3.10 constraint. + +Using version managers (pyenv, mise, asdf) +========================================== + +virtualenv automatically resolves shims from `pyenv `_, `mise `_, +and `asdf `_ to the real Python binary. Set the active Python version using any of the standard +mechanisms and virtualenv will discover it: + +.. code-block:: console + + $ pyenv local 3.12.0 + $ virtualenv venv # uses pyenv's 3.12.0, not the system Python + + $ PYENV_VERSION=3.11.0 virtualenv venv # uses 3.11.0 + +This also works with mise and asdf: + +.. code-block:: console + + $ mise use python@3.12 + $ virtualenv venv + +No additional configuration is required. See :doc:`../explanation` for details on how shim resolution works. + +******************************** + Activate a virtual environment +******************************** + +Activate the environment to modify your shell's PATH and environment variables. + +.. tab:: Bash/Zsh + + .. code-block:: console + + $ source venv/bin/activate + +.. tab:: Fish + + .. code-block:: console + + $ source venv/bin/activate.fish + +.. tab:: PowerShell + + .. code-block:: console + + PS> .\venv\Scripts\Activate.ps1 + + .. note:: + + If you encounter an execution policy error, run ``Set-ExecutionPolicy RemoteSigned`` to allow local scripts. + +.. tab:: CMD + + .. code-block:: console + + > .\venv\Scripts\activate.bat + +.. tab:: Nushell + + .. code-block:: console + + $ overlay use venv/bin/activate.nu + +.. tab:: Xonsh + + .. code-block:: console + + @ source venv/bin/activate.xsh + +Deactivate the environment +========================== + +Exit the virtual environment: + +.. code-block:: console + + $ deactivate + +Use without activation +====================== + +Use the environment without activating it by calling executables with their full paths: + +.. code-block:: console + + $ venv/bin/python script.py + $ venv/bin/pip install package + +``python``, ``python3`` and the versioned name reach the same interpreter, and on Windows the executables live in +``venv\Scripts`` under ``.exe`` names. Reach for ``pythonw.exe`` there when you want a process without a console window. +:doc:`../reference/environment-layout` lists every name. + +Customize prompt +================ + +Set a custom prompt prefix: + +.. code-block:: console + + $ virtualenv --prompt myproject venv + +Disable the prompt modification by setting the ``VIRTUAL_ENV_DISABLE_PROMPT`` environment variable. + +Access the prompt string via the ``VIRTUAL_ENV_PROMPT`` environment variable. + +Programmatic activation +======================= + +Activate the environment from within a running Python process using ``activate_this.py``. This modifies ``sys.path`` and +environment variables in the current process so that subsequent imports resolve from the virtual environment. + +.. code-block:: python + + import runpy + + runpy.run_path("venv/bin/activate_this.py") + +A common use case is web applications served by a system-wide WSGI server (such as mod_wsgi or uWSGI) that need to load +packages from a virtual environment: + +.. code-block:: python + + import runpy + from pathlib import Path + + runpy.run_path(str(Path("/var/www/myapp/venv/bin/activate_this.py"))) + + from myapp import create_app # noqa: E402 + + application = create_app() + +******************** + Configure defaults +******************** + +Use a configuration file to set default options for virtualenv. + +Configuration file location +=========================== + +The configuration file is named ``virtualenv.ini`` and located in the platformdirs app config directory. Run +``virtualenv --help`` to see the exact location for your system. + +Override the location with the ``VIRTUALENV_CONFIG_FILE`` environment variable. + +Configuration format +==================== + +Derive configuration keys from command-line options by stripping leading ``-`` and replacing remaining ``-`` with ``_``: + +.. code-block:: ini + + [virtualenv] + python = /opt/python-3.14/bin/python + +Multi-value options +=================== + +Specify multiple values on separate lines: + +.. code-block:: ini + + [virtualenv] + extra_search_dir = + /path/to/dists + /path/to/other/dists + +Environment variables +===================== + +Set options using environment variables with the ``VIRTUALENV_`` prefix and uppercase key names: + +.. code-block:: console + + $ export VIRTUALENV_PYTHON=/opt/python-3.14/bin/python + +For multi-value options, separate values with commas or newlines. + +Override app-data location +========================== + +Set the ``VIRTUALENV_OVERRIDE_APP_DATA`` environment variable to override the default app-data cache directory location. + +Allow unverified HTTPS for periodic updates +=========================================== + +The periodic update checks for newer seed wheels by fetching ``https://pypi.org/pypi//json``. The request +is made with full TLS verification and, if the verified request fails, virtualenv now skips the update rather than +downgrading the connection. Set ``VIRTUALENV_PERIODIC_UPDATE_INSECURE=1`` to restore the previous behavior and retry +with an unverified SSL context. This is an escape hatch for hosts with broken trust stores; do not set it on hosts you +do not control. + +Configuration priority +====================== + +Options are resolved in this order (highest to lowest priority): + +.. mermaid:: + + block-beta + columns 1 + A["Command-line arguments (highest)"] + B["Environment variables"] + C["Configuration file"] + D["Default values (lowest)"] + + style A fill:#16a34a,stroke:#15803d,color:#fff + style B fill:#2563eb,stroke:#1d4ed8,color:#fff + style C fill:#d97706,stroke:#b45309,color:#fff + style D fill:#6366f1,stroke:#4f46e5,color:#fff + +******************************** + Make environments discoverable +******************************** + +In a project folder, one holding a ``pyproject.toml``, virtualenv points a ``.venv`` redirect file at the first +environment you create there, so editors and type checkers can find it without an activated shell. See `PEP 832 +`_ for the format and :ref:`explanation:Environment discovery` for the reasoning. The +feature is provisional while the PEP is a draft, so a minor or patch release may change it in backward incompatible +ways. + +Point a tool at the right environment +===================================== + +The redirect names the first environment you created in the project: + +.. code-block:: console + + $ virtualenv py313 --python 3.13 + $ virtualenv py314 --python 3.14 + $ cat .venv + py313 + +Pass ``--venv-redirect`` to make another environment the default, or to write the redirect outside a project folder: + +.. code-block:: console + + $ virtualenv py314 --python 3.14 --venv-redirect + $ cat .venv + py314 + +virtualenv leaves a ``.venv`` folder alone, and a redirect pointing at an environment it did not create. Delete the +``.venv`` if you want virtualenv to take it over. + +Choose once for every environment +================================= + +Set the choice in the configuration file or an environment variable. ``no_venv_redirect = true`` stops the redirect: + +.. code-block:: ini + + [virtualenv] + no_venv_redirect = true + +.. code-block:: console + + $ export VIRTUALENV_NO_VENV_REDIRECT=1 + +``venv_redirect = true`` or ``VIRTUALENV_VENV_REDIRECT=1`` writes it for every environment, as ``--venv-redirect`` does. +A flag on the command line beats both, so ``virtualenv env --venv-redirect`` writes the file for one run even with +``no_venv_redirect = true`` set. + +Commit the file or ignore it +============================ + +The redirect holds a path relative to its own folder, so you can commit it when the environment location is the same for +everyone on the project, such as a containerized setup. Leave ``.venv`` in ``.gitignore`` when developers pick their own +paths. + +*********************** + Control seed packages +*********************** + +Upgrade embedded wheels +======================= + +Update the embedded wheel files to the latest versions: + +.. code-block:: console + + $ virtualenv --upgrade-embed-wheels + +Provide custom wheels +===================== + +Use custom wheel files from a local directory: + +.. code-block:: console + + $ virtualenv --extra-search-dir /path/to/wheels venv + +Download latest from PyPI +========================= + +Download the latest versions of seed packages from PyPI: + +.. code-block:: console + + $ virtualenv --download venv + +Disable periodic updates +======================== + +Disable automatic periodic updates of seed packages: + +.. code-block:: console + + $ virtualenv --no-periodic-update venv + +For distribution maintainers +============================ + +Patch the ``virtualenv.seed.wheels.embed`` module and set ``PERIODIC_UPDATE_ON_BY_DEFAULT`` to ``False`` to disable +periodic updates by default. See :doc:`../explanation` for implementation details. + +******************************************* + Find out which Python an environment uses +******************************************* + +Read ``python-version`` out of ``pyvenv.cfg`` rather than running the environment's interpreter, which costs a +subprocess and fails when the base Python has been removed: + +.. code-block:: python + + from configparser import ConfigParser + from pathlib import Path + + parser = ConfigParser() + parser.read_string("[cfg]\n" + Path("env/pyvenv.cfg").read_text(encoding="utf-8")) + parser["cfg"]["python-version"] # '3.14' + +``pyvenv.cfg`` has no section header, hence the prefix. The key holds the feature release only; reach for ``version`` +when you need the patch level too. + +********************** + Use from Python code +********************** + +Call virtualenv from Python code using the ``cli_run`` function: + +.. code-block:: python + + from virtualenv import cli_run + + cli_run(["venv"]) + +Pass options as list elements: + +.. code-block:: python + + cli_run(["-p", "python3.14", "--without-pip", "myenv"]) + +Use the returned session object to access environment details: + +.. code-block:: python + + result = cli_run(["venv"]) + print(result.creator.dest) # path to created environment + print(result.creator.exe) # path to python executable + +Use ``session_via_cli`` to describe the environment without creating it: + +.. code-block:: python + + from virtualenv import session_via_cli + + session = session_via_cli(["venv"]) + # inspect session.creator, session.seeder, session.activators + +See :doc:`../reference/api` for complete API documentation. diff --git a/docs/how-to/verify-release.rst b/docs/how-to/verify-release.rst new file mode 100644 index 000000000..2bb5a5d4b --- /dev/null +++ b/docs/how-to/verify-release.rst @@ -0,0 +1,240 @@ +############################# + Verify a virtualenv release +############################# + +The ``release.yaml`` workflow in `pypa/virtualenv `_ builds and publishes every +release. The steps below check that a file you downloaded came out of that workflow, and show you what the release +bundles. The examples use release ``21.12.1``; replace it with the version you have. +:doc:`../reference/release-artifacts` lists every file a release publishes, and :ref:`release-integrity` covers what +these checks prove. + +You need `uv `_, whose ``uvx`` runs ``pip`` and `pypi-attestations +`_ without installing them, the `GitHub CLI `_ for +the GitHub release assets and attestations, and `jq `_ to read the SBOMs. + +*********************************** + Verify a wheel or sdist from PyPI +*********************************** + +Download the wheel and the sdist without installing them: + +.. code-block:: console + + $ uvx pip download virtualenv==21.12.1 --no-deps --dest . + $ uvx pip download virtualenv==21.12.1 --no-deps --no-binary :all: --dest . + +Check each file against the attestation PyPI stores for it, one file per call: + +.. code-block:: console + + $ uvx pypi-attestations verify pypi --repository https://github.com/pypa/virtualenv virtualenv-21.12.1-py3-none-any.whl + OK: virtualenv-21.12.1-py3-none-any.whl + $ uvx pypi-attestations verify pypi --repository https://github.com/pypa/virtualenv virtualenv-21.12.1.tar.gz + OK: virtualenv-21.12.1.tar.gz + +A modified file fails with ``subject does not match distribution digest``, and a file signed by another repository fails +with ``provenance was signed by repository ...``. To check the copy on PyPI without downloading it first, prefix the +file name with ``pypi:``, as in ``pypi:virtualenv-21.12.1-py3-none-any.whl``. + +************************************** + Verify the GitHub release and assets +************************************** + +Releases from 21.11.0 on are `immutable +`_: +GitHub signs a release attestation that binds the tag to its commit and to the SHA-256 of every asset, and refuses any +later change to the tag or the assets. Check that attestation: + +.. code-block:: console + + $ gh release verify 21.12.1 --repo pypa/virtualenv + Resolved tag 21.12.1 to sha1:befec5eae075d1c4cb00a41d0c72bcdb91bf4586 + Loaded attestation from GitHub API + ✓ Release 21.12.1 verified! + +Then check a file you downloaded against the digests the attestation lists. The command hashes the local file, so it +works for a copy under any name: + +.. code-block:: console + + $ gh release download 21.12.1 --repo pypa/virtualenv --pattern virtualenv.pyz + $ gh release verify-asset 21.12.1 virtualenv.pyz --repo pypa/virtualenv + ✓ Verification succeeded! virtualenv.pyz is present in release 21.12.1 + +GitHub signs the release attestation itself, so it shows the assets did not change after publication. It does not name +the workflow that built them; the provenance check below does. + +******************* + Verify the zipapp +******************* + +Download the zipapp and its provenance bundle from the GitHub release, then verify one against the other: + +.. code-block:: console + + $ gh release download 21.12.1 --repo pypa/virtualenv --pattern virtualenv.pyz --pattern virtualenv.pyz.intoto.jsonl + $ gh attestation verify virtualenv.pyz --repo pypa/virtualenv --bundle virtualenv.pyz.intoto.jsonl \ + --signer-workflow pypa/virtualenv/.github/workflows/release.yaml + +Add ``--source-ref refs/tags/21.12.1`` to also require that the build ran from the ``21.12.1`` tag. Without +``--bundle``, ``gh`` fetches the attestation from GitHub instead of reading the local file. + +The zipapp at ``https://bootstrap.pypa.io/virtualenv.pyz`` comes from `pypa/get-virtualenv +`_ and can trail the latest release. ``python virtualenv.pyz --version`` shows +which release you have. Verify it with ``gh release verify-asset`` for that release, or with ``gh attestation verify`` +as above, leaving out ``--bundle``. Releases before 21.7.11 have neither a release attestation nor a provenance bundle; +compare the file's SHA-256 with the digest GitHub lists for the release asset instead: + +.. code-block:: console + + $ gh release view 21.7.10 --repo pypa/virtualenv --json assets --jq '.assets[] | .name + " " + .digest' + virtualenv.pyz sha256:06ee4ea84517e9b8565f7ee81c064d2de5e83dad3874d2babd7b94b2b40c4595 + $ shasum -a 256 virtualenv.pyz + 06ee4ea84517e9b8565f7ee81c064d2de5e83dad3874d2babd7b94b2b40c4595 virtualenv.pyz + +************************ + Read the embedded SBOM +************************ + +The wheel carries a `CycloneDX `_ SBOM that lists the ``pip`` and ``setuptools`` wheels +virtualenv bundles, with their hashes, licenses and the Python versions each one seeds. Extract it from the wheel: + +.. code-block:: console + + $ python -m zipfile --extract virtualenv-21.12.1-py3-none-any.whl wheel + $ jq -r '.components[] | select(.hashes) | "\(.name) \(.version) Python \([.properties[] | select(.name == "virtualenv:seeded-for-python").value] | join(","))"' \ + wheel/virtualenv-21.12.1.dist-info/sboms/virtualenv.cdx.json + pip 26.0.1 Python 3.9 + pip 26.2.1 Python 3.10,3.11,3.12,3.13,3.14,3.15,3.16 + setuptools 82.0.1 Python 3.9 + setuptools 84.0.0 Python 3.10,3.11,3.12,3.13,3.14,3.15,3.16 + +An installed virtualenv keeps the same file in its ``.dist-info`` directory. Print its path with the Python that runs +virtualenv: + +.. code-block:: console + + $ python -c "import importlib.metadata as m; print(next(f.locate() for f in m.files('virtualenv') if f.name == 'virtualenv.cdx.json'))" + +****************************** + Verify the SBOM attestations +****************************** + +The release workflow attests the SBOM against the wheel and the sdist. Check that attestation exists and was signed by +the release workflow: + +.. code-block:: console + + $ gh attestation verify virtualenv-21.12.1-py3-none-any.whl -R pypa/virtualenv --predicate-type https://cyclonedx.org/bom + +To confirm the attested SBOM matches the one inside the wheel, save the attested copy and compare the two. ``diff`` +prints nothing when they match: + +.. code-block:: console + + $ gh attestation verify virtualenv-21.12.1-py3-none-any.whl -R pypa/virtualenv --predicate-type https://cyclonedx.org/bom \ + --format json --jq '.[0].verificationResult.statement.predicate' > attested.cdx.json + $ diff <(jq -S . attested.cdx.json) <(jq -S . wheel/virtualenv-21.12.1.dist-info/sboms/virtualenv.cdx.json) + +Releases from 21.11.0 on also attach the SBOM as ``virtualenv.cdx.json`` and an SPDX 2.3 rendering of it as +``virtualenv.spdx.json``, and attest the SPDX document against the wheel and the sdist as well. The CycloneDX asset must +match the SBOM in the wheel, and the SPDX asset what the release attested: + +.. code-block:: console + + $ gh release download 21.12.1 --repo pypa/virtualenv --pattern virtualenv.cdx.json --pattern virtualenv.spdx.json + $ unzip -p virtualenv-21.12.1-py3-none-any.whl '*.dist-info/sboms/virtualenv.cdx.json' | cmp - virtualenv.cdx.json + $ gh attestation verify virtualenv-21.12.1-py3-none-any.whl -R pypa/virtualenv --predicate-type https://spdx.dev/Document/v2.3 \ + --format json --jq '.[0].verificationResult.statement.predicate' > attested.spdx.json + $ diff <(jq -S . attested.spdx.json) <(jq -S . virtualenv.spdx.json) + +********************** + Read the zipapp SBOM +********************** + +Releases from 21.11.0 on describe the zipapp in its own CycloneDX SBOM. The zipapp carries it at its root, the release +attaches it as ``virtualenv.pyz.cdx.json``, and GitHub attests it against ``virtualenv.pyz``. Check all three agree: + +.. code-block:: console + + $ gh release download 21.12.1 --repo pypa/virtualenv --pattern virtualenv.pyz --pattern virtualenv.pyz.cdx.json + $ unzip -p virtualenv.pyz virtualenv.pyz.cdx.json | cmp - virtualenv.pyz.cdx.json + $ gh attestation verify virtualenv.pyz -R pypa/virtualenv --predicate-type https://cyclonedx.org/bom \ + --format json --jq '.[0].verificationResult.statement.predicate' > attested.pyz.cdx.json + $ diff <(jq -S . attested.pyz.cdx.json) <(jq -S . virtualenv.pyz.cdx.json) + +List the distributions the zipapp bundles and the Python versions that load each one: + +.. code-block:: console + + $ jq -r '.components[] | "\(.name) \(.version) \([.properties[] | select(.name == "virtualenv:loaded-for-python").value] | join(","))"' \ + virtualenv.pyz.cdx.json + virtualenv 21.12.1 + distlib 0.4.3 3.14,3.13,3.12,3.11,3.10,3.9,3.8 + filelock 3.19.1 3.9,3.8 + filelock 3.32.6 3.14,3.13,3.12,3.11,3.10 + platformdirs 4.11.8 3.14,3.13,3.12,3.11,3.10 + platformdirs 4.4.0 3.9,3.8 + python-discovery 1.6.0 3.14,3.13,3.12,3.11,3.10,3.9,3.8 + typing_extensions 4.16.0 3.10,3.9,3.8 + +*************************** + Rebuild the release files +*************************** + +The release builds with ``SOURCE_DATE_EPOCH`` set to the commit time of the release tag, so rebuilding the tag yields +the same sdist, byte for byte: + +.. code-block:: console + + $ git clone --branch 21.12.1 https://github.com/pypa/virtualenv + $ cd virtualenv + $ SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) uv build --sdist --out-dir rebuild . + $ cmp rebuild/virtualenv-21.12.1.tar.gz ../virtualenv-21.12.1.tar.gz + +``cmp`` prints nothing when the files match. + +The wheel of a release from 21.11.0 on rebuilds byte for byte too, on any operating system and architecture, once the +Python patch version and the build backend versions match the ones the release used. Its SBOM lists both, so read them +from the published wheel and pass them to the build: + +.. code-block:: console + + $ unzip -p ../virtualenv-21.12.1-py3-none-any.whl '*.dist-info/sboms/virtualenv.cdx.json' > published.cdx.json + $ jq -r '.metadata.tools.components[] | select(.type == "platform") | .version' published.cdx.json + 3.14.7 + $ jq -r '.metadata.tools.components[] | select(.purl // "" | startswith("pkg:pypi/")) | "\(.name)==\(.version)"' \ + published.cdx.json > build-constraints.txt + $ SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) uv build --wheel --python 3.14.7 \ + --build-constraint build-constraints.txt --out-dir rebuild . + $ cmp rebuild/virtualenv-21.12.1-py3-none-any.whl ../virtualenv-21.12.1-py3-none-any.whl + +Releases after 21.13.0 also pin the backend in ``tasks/release-requirements.txt``, so on those tags +``--build-constraints tasks/release-requirements.txt --require-hashes`` replaces the file read from the SBOM. Build from +a git checkout, since the SBOM records the source commit and an sdist does not carry it. Wheels up to 21.10.0 recorded +the machine that built them in the SBOM, so a rebuild of those differs in the SBOM and in ``RECORD``, which holds the +SBOM's hash. + +The zipapp of a release from 21.11.0 on rebuilds byte for byte as well. The `pylock.zipapp.toml +`_ lock at the tag pins each distribution it bundles by +version and SHA-256, so only the build tools can drift. Two SBOMs list them: the zipapp SBOM holds the tools of the +``tox`` environment that assembles the zipapp, and the wheel SBOM inside the zipapp holds the build backend for that +wheel. Constrain both, and run on the CPython version the zipapp SBOM lists: + +.. code-block:: console + + $ jq -r '.metadata.tools.components[] | select(.purl // "" | startswith("pkg:pypi/")) | "\(.name)==\(.version)"' \ + ../virtualenv.pyz.cdx.json > zipapp-constraints.txt + $ unzip -p ../virtualenv.pyz 'virtualenv-*.dist-info/sboms/virtualenv.cdx.json' \ + | jq -r '.metadata.tools.components[] | select(.purl // "" | startswith("pkg:pypi/")) | "\(.name)==\(.version)"' \ + > wheel-constraints.txt + $ UV_CONSTRAINT=$PWD/zipapp-constraints.txt PIP_BUILD_CONSTRAINT=$PWD/wheel-constraints.txt \ + SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) uvx --with tox-uv tox r -e zipapp -x 'env.zipapp.pass_env+=PIP_BUILD_CONSTRAINT' + $ cmp virtualenv.pyz ../virtualenv.pyz + +The release runs ``tox`` with the ``tox-uv`` plugin as well. The plugin passes ``UV_*`` variables into the environment, +and the ``-x`` override adds ``PIP_BUILD_CONSTRAINT`` for the ``pip wheel`` call that builds the virtualenv wheel inside +the zipapp. + +Releases after 21.13.0 pin those build tools and the SBOM tools in ``tasks/release-requirements.txt`` at the tag, so a +rebuild of those takes its constraints from that file and needs neither SBOM. diff --git a/docs/index.rst b/docs/index.rst index 52484a9ae..b868311ee 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,122 +1,145 @@ -virtualenv -========== +############ + virtualenv +############ .. image:: https://img.shields.io/pypi/v/virtualenv?style=flat-square - :target: https://pypi.org/project/virtualenv/#history - :alt: Latest version on PyPI + :target: https://pypi.org/project/virtualenv/#history + :alt: Latest version on PyPI + .. image:: https://img.shields.io/pypi/implementation/virtualenv?style=flat-square - :alt: PyPI - Implementation + :alt: PyPI - Implementation + .. image:: https://img.shields.io/pypi/pyversions/virtualenv?style=flat-square - :alt: PyPI - Python Version + :alt: PyPI - Python Version + .. image:: https://readthedocs.org/projects/virtualenv/badge/?version=latest&style=flat-square - :target: https://virtualenv.pypa.io - :alt: Documentation status + :target: https://virtualenv.pypa.io + :alt: Documentation status + .. image:: https://img.shields.io/discord/803025117553754132 - :target: https://discord.gg/pypa - :alt: Discord + :target: https://discord.gg/pypa + :alt: Discord + .. image:: https://img.shields.io/pypi/dm/virtualenv?style=flat-square - :target: https://pypistats.org/packages/virtualenv - :alt: PyPI - Downloads + :target: https://pypistats.org/packages/virtualenv + :alt: PyPI - Downloads + .. image:: https://img.shields.io/pypi/l/virtualenv?style=flat-square - :target: https://opensource.org/licenses/MIT - :alt: PyPI - License + :target: https://opensource.org/licenses/MIT + :alt: PyPI - License + .. image:: https://img.shields.io/github/issues/pypa/virtualenv?style=flat-square - :target: https://github.com/pypa/virtualenv/issues - :alt: Open issues + :target: https://github.com/pypa/virtualenv/issues + :alt: Open issues + .. image:: https://img.shields.io/github/issues-pr/pypa/virtualenv?style=flat-square - :target: https://github.com/pypa/virtualenv/pulls - :alt: Open pull requests + :target: https://github.com/pypa/virtualenv/pulls + :alt: Open pull requests + .. image:: https://img.shields.io/github/stars/pypa/virtualenv?style=flat-square - :target: https://pypistats.org/packages/virtualenv - :alt: Package popularity + :target: https://pypistats.org/packages/virtualenv + :alt: Package popularity -``virtualenv`` is a tool to create isolated Python environments. +``virtualenv`` is a tool to create isolated Python environments. Since Python 3.3, a subset of it has been integrated +into the standard library under the ``venv`` module. For how ``virtualenv`` compares to the stdlib ``venv`` module, see +:doc:`explanation`. -virtualenv vs venv ------------------- +****************** + Quick navigation +****************** -Since Python ``3.3``, a subset of it has been -integrated into the standard library under the `venv module `_. The -``venv`` module does not offer all features of this library, to name just a few more prominent: +**Tutorials** - Learn by doing -- is slower (by not having the ``app-data`` seed method), -- is not as extendable, -- cannot create virtual environments for arbitrarily installed python versions (and automatically discover these), -- is not upgrade-able via `pip `_, -- does not have as rich programmatic API (describe virtual environments without creating them). +- :doc:`tutorial/getting-started` — Create your first virtual environment and learn the basic workflow -Concept and purpose of virtualenv ---------------------------------- +**How-to guides** - Solve specific problems -The basic problem being addressed is one of dependencies and versions, and indirectly permissions. -Imagine you have an application that needs version ``1`` of ``LibFoo``, but another application requires version -``2``. How can you use both these libraries? If you install everything into your host python (e.g. ``python3.8``) -it's easy to end up in a situation where two packages have conflicting requirements. +- :doc:`how-to/install` — Install virtualenv on your system +- :doc:`how-to/usage` — Select Python versions, activate environments, configure defaults, and use from Python code +- :doc:`how-to/verify-release` — Check that a downloaded release came from the virtualenv release workflow -Or more generally, what if you want to install an application *and leave it be*? If an application works, any change -in its libraries or the versions of those libraries can break the application. Also, what if you can't install packages -into the global ``site-packages`` directory, due to not having permissions to change the host python environment? +**Reference** - Technical information -In all these cases, ``virtualenv`` can help you. It creates an environment that has its own installation directories, -that doesn't share libraries with other virtualenv environments (and optionally doesn't access the globally installed -libraries either). +- :doc:`reference/compatibility` — Supported Python versions and operating systems +- :doc:`reference/cli` — Command line options and flags +- :doc:`reference/environment-layout` — Names the interpreter answers to inside an environment +- :doc:`reference/files` — Files written inside and beside a created environment +- :doc:`reference/api` — Programmatic Python API reference +- :doc:`reference/release-artifacts` — Files, attestations and SBOMs each release publishes +**Explanation** - Understand the concepts -Compatibility -------------- +- :doc:`explanation` — How virtualenv works under the hood and why it exists -With the release of virtualenv 20.22, April 2023, (`release note `__) target interpreters are now limited to Python v. 3.7+. +**Extensions** -Trying to use an earlier version will normally result in the target interpreter raising a syntax error. This virtualenv tool will then print some details about the exception and abort, ie no explicit warning about trying to use an outdated/incompatible version. It may look like this: +- :doc:`plugin/index` — Extend virtualenv with custom creators, seeders, and activators -.. code-block:: console +****************** + Related projects +****************** - $ virtualenv --discovery pyenv -p python3.6 foo - RuntimeError: failed to query /home/velle/.pyenv/versions/3.6.15/bin/python3.6 with code 1 err: ' File "/home/velle/.virtualenvs/toxrunner/lib/python3.12/site-packages/virtualenv/discovery/py_info.py", line 7 - from __future__ import annotations - ^ - SyntaxError: future feature annotations is not defined +Several tools build on virtualenv to provide higher-level workflows: +- `virtualenvwrapper `_ — Shell wrapper for creating and managing + multiple virtualenvs +- `pew `_ — Python Env Wrapper, a set of commands to manage multiple virtual + environments +- `tox `_ — Automate testing across multiple Python versions +- `nox `_ — Flexible test automation in Python -In tox, even if the interpreter is installed and available, the message is (somewhat misleading): +******************** + External resources +******************** -.. code-block:: console +Learn more about virtualenv from these community resources: - py36: skipped because could not find python interpreter with spec(s): py36 +- `Corey Schafer's virtualenv tutorial `_ — Video walkthrough for beginners +- `Bernat Gabor's status quo `_ — Talk about the current state of Python + packaging +- `Carl Meyer's reverse-engineering `_ + — Deep dive into how virtualenv works internally +.. toctree:: + :hidden: + :caption: Tutorial + tutorial/getting-started -Useful links ------------- +.. toctree:: + :hidden: + :caption: How-to guides -**Related projects, that build abstractions on top of virtualenv** + how-to/install + how-to/usage + how-to/verify-release -* :pypi:`virtualenvwrapper` - a useful set of scripts for creating and deleting virtual environments -* :pypi:`pew` - provides a set of commands to manage multiple virtual environments -* :pypi:`tox` - a generic virtualenv management and test automation command line tool, driven by a ``tox.ini`` - configuration file -* :pypi:`nox` - a tool that automates testing in multiple Python environments, similar to tox, - driven by a ``noxfile.py`` configuration file +.. toctree:: + :hidden: + :caption: Reference -**Tutorials** + reference/compatibility + reference/cli + reference/environment-layout + reference/files + reference/api + reference/release-artifacts -* `Corey Schafer tutorial `_ on how to use it -* `Using virtualenv with mod_wsgi `_ +.. toctree:: + :hidden: + :caption: Explanation -**Presenting how the package works from within** + explanation -* `Bernat Gabor: status quo of virtual environments `_ -* `Carl Meyer: Reverse-engineering Ian Bicking's brain: inside pip and virtualenv - `_ +.. toctree:: + :hidden: + :caption: Extend -.. comment: split here + plugin/index .. toctree:: - :hidden: - - installation - user_guide - cli_interface - extend - development - changelog + :hidden: + :caption: Project + + development + changelog diff --git a/docs/installation.rst b/docs/installation.rst deleted file mode 100644 index b82d1dd19..000000000 --- a/docs/installation.rst +++ /dev/null @@ -1,124 +0,0 @@ -Installation -============ - -via pipx --------- - -:pypi:`virtualenv` is a CLI tool that needs a Python interpreter to run. If you already have a ``Python 3.7+`` -interpreter the best is to use :pypi:`pipx` to install virtualenv into an isolated environment. This has the added -benefit that later you'll be able to upgrade virtualenv without affecting other parts of the system. - -.. code-block:: console - - pipx install virtualenv - virtualenv --help - -via pip -------- - -Alternatively you can install it within the global Python interpreter itself (perhaps as a user package via the -``--user`` flag). Be cautious if you are using a python install that is managed by your operating system or -another package manager. ``pip`` might not coordinate with those tools, and may leave your system in an -inconsistent state. Note, if you go down this path you need to ensure pip is new enough per the subsections below: - -.. code-block:: console - - python -m pip install --user virtualenv - python -m virtualenv --help - -wheel -~~~~~ -Installing virtualenv via a wheel (default with pip) requires an installer that can understand the ``python-requires`` -tag (see `PEP-503 `_), with pip this is version ``9.0.0`` (released 2016 -November). Furthermore, in case you're not installing it via the PyPi you need to be using a mirror that correctly -forwards the ``python-requires`` tag (notably the OpenStack mirrors don't do this, or older -`devpi `_ versions - added with version ``4.7.0``). - -.. _sdist: - -sdist -~~~~~ -When installing via a source distribution you need an installer that handles the -`PEP-517 `_ specification. In case of ``pip`` this is version ``18.0.0`` or -later (released on 2018 July). If you cannot upgrade your pip to support this you need to ensure that the build -requirements from `pyproject.toml `_ are satisfied -before triggering the install. - -via zipapp ----------- - -You can use virtualenv without installing it too. We publish a Python -`zipapp `_, you can just download this from -`https://bootstrap.pypa.io/virtualenv.pyz `_ and invoke this package -with a python interpreter: - -.. code-block:: console - - python virtualenv.pyz --help - -The root level zipapp is always the current latest release. To get the last supported zipapp against a given python -minor release use the link ``https://bootstrap.pypa.io/virtualenv/x.y/virtualenv.pyz``, e.g. for the last virtualenv -supporting Python 3.11 use -`https://bootstrap.pypa.io/virtualenv/3.11/virtualenv.pyz `_. - -If you are looking for past version of virtualenv.pyz they are available here: - -.. code-block:: console - - https://github.com/pypa/get-virtualenv/blob//public//virtualenv.pyz?raw=true - -latest unreleased ------------------ -Installing an unreleased version is discouraged and should be only done for testing purposes. If you do so you'll need -a pip version of at least ``18.0.0`` and use the following command: - - -.. code-block:: console - - pip install git+https://github.com/pypa/virtualenv.git@main - -.. _compatibility-requirements: - -Python and OS Compatibility ---------------------------- - -virtualenv works with the following Python interpreter implementations: - -- `CPython `_: ``3.13 >= python_version >= 3.7`` -- `PyPy `_: ``3.10 >= python_version >= 3.7`` - -This means virtualenv works on the latest patch version of each of these minor versions. Previous patch versions are -supported on a best effort approach. - -CPython is shipped in multiple forms, and each OS repackages it, often applying some customization along the way. -Therefore we cannot say universally that we support all platforms, but rather specify some we test against. In case -of ones not specified here the support is unknown, though likely will work. If you find some cases please open a feature -request on our issue tracker. - -Note: - -- as of ``20.27.0`` -- ``2024-10-17`` -- we no longer support running under Python ``<=3.7``, -- as of ``20.18.0`` -- ``2023-02-06`` -- we no longer support running under Python ``<=3.6``, -- as of ``20.22.0`` -- ``2023-04-19`` -- we no longer support creating environments for Python ``<=3.6``. - -Linux -~~~~~ -- installations from `python.org `_ -- Ubuntu 16.04+ (both upstream and `deadsnakes `_ builds) -- Fedora -- RHEL and CentOS -- OpenSuse -- Arch Linux - -macOS -~~~~~ -In case of macOS we support: - -- installations from `python.org `_, -- python versions installed via `brew `_, -- Python 3 part of XCode (Python framework - ``/Library/Frameworks/Python3.framework/``). - -Windows -~~~~~~~ -- Installations from `python.org `_ -- Windows Store Python - note only `version 3.8+ `_ diff --git a/docs/plugin/api.rst b/docs/plugin/api.rst new file mode 100644 index 000000000..dee75e40e --- /dev/null +++ b/docs/plugin/api.rst @@ -0,0 +1,79 @@ +###################### + Plugin API reference +###################### + +This page documents the interfaces that plugins must implement. + +*********** + Discovery +*********** + +Discovery plugins locate Python interpreters for creating virtual environments. + +.. currentmodule:: virtualenv.discovery.discover + +.. autoclass:: Discover + :undoc-members: + :members: + +PythonInfo +========== + +Discovery plugins return a ``PythonInfo`` object describing the located interpreter. + +.. currentmodule:: virtualenv.discovery.py_info + +.. autoclass:: PythonInfo + :undoc-members: + :members: + +********** + App data +********** + +The application data interface used by plugins for caching. + +.. currentmodule:: virtualenv.app_data.base + +.. autoclass:: AppData + :members: + +********** + Creators +********** + +Creator plugins build the virtual environment directory structure and install the Python interpreter. + +.. currentmodule:: virtualenv.create.creator + +.. autoclass:: CreatorMeta + :members: + +.. autoclass:: Creator + :undoc-members: + :members: + :exclude-members: run, set_pyenv_cfg, debug_script, validate_dest, debug + +********* + Seeders +********* + +Seeder plugins install initial packages (like pip, setuptools, wheel) into the virtual environment. + +.. currentmodule:: virtualenv.seed.seeder + +.. autoclass:: Seeder + :undoc-members: + :members: + +************ + Activators +************ + +Activator plugins generate shell-specific activation scripts. + +.. currentmodule:: virtualenv.activation.activator + +.. autoclass:: Activator + :undoc-members: + :members: diff --git a/docs/plugin/architecture.rst b/docs/plugin/architecture.rst new file mode 100644 index 000000000..4ce911622 --- /dev/null +++ b/docs/plugin/architecture.rst @@ -0,0 +1,125 @@ +##################### + Plugin architecture +##################### + +This page explains how virtualenv's plugin system works internally. + +************** + Entry points +************** + +virtualenv uses Python entry points (``setuptools`` / ``importlib.metadata``) to discover plugins. Each plugin registers +under one of four entry point groups: + +- ``virtualenv.discovery`` +- ``virtualenv.create`` +- ``virtualenv.seed`` +- ``virtualenv.activate`` + +At startup, virtualenv loads all registered entry points from these groups and makes them available as CLI options. +Built-in implementations are registered in virtualenv's own ``pyproject.toml``, while third-party plugins register their +entry points in their own package metadata. + +When a package with virtualenv plugins is installed in the same environment as virtualenv, the plugins become +immediately available without additional configuration. + +****************** + Plugin lifecycle +****************** + +The following diagram shows how plugins are discovered and executed: + +.. mermaid:: + + sequenceDiagram + participant User + participant CLI + participant EntryPoints + participant Discovery + participant Creator + participant Seeder + participant Activator + + rect rgba(37, 99, 235, 0.15) + User->>CLI: virtualenv myenv + CLI->>EntryPoints: Load plugins from all groups + EntryPoints-->>CLI: Available plugins + CLI->>CLI: Build argument parser with plugin options + CLI->>User: Parse CLI arguments + User-->>CLI: Selected options + end + + rect rgba(22, 163, 106, 0.15) + CLI->>Discovery: Run selected discovery plugin + Discovery-->>CLI: PythonInfo + CLI->>Creator: Create environment with PythonInfo + Creator-->>CLI: Created environment + CLI->>Seeder: Seed packages into environment + Seeder-->>CLI: Seeded environment + CLI->>Activator: Generate activation scripts + Activator-->>CLI: Complete environment + end + +The lifecycle follows these stages: + +1. virtualenv starts and discovers all entry points from the four plugin groups +2. The CLI parser is built dynamically, incorporating options from all discovered plugins +3. User arguments are parsed to select which discovery, creator, seeder, and activator plugins to use +4. Selected plugins execute in sequence: discover → create → seed → activate +5. Each stage passes its output to the next stage + +************************ + Extension point design +************************ + +Each extension point follows a consistent pattern: + +Base abstract class + Each extension point defines a base abstract class (``Discover``, ``Creator``, ``Seeder``, ``Activator``) that + specifies the interface plugins must implement. + +Built-in implementations + virtualenv includes built-in implementations registered as entry points in its own ``pyproject.toml``. For example, + the built-in CPython creator is registered as ``cpython3-posix``. + +Third-party plugins + External packages implement the base interface and register their own entry points under the same group. When + installed, they appear alongside built-in options. + +CLI selection + Command-line flags (``--discovery``, ``--creator``, ``--seeder``, ``--activators``) allow users to select which + implementation to use. Multiple activators can be selected simultaneously. + +Parser integration + Each plugin can contribute CLI arguments through the ``add_parser_arguments`` classmethod. These arguments appear in + ``virtualenv --help`` and are available when the plugin is selected. + +********************** + How plugins interact +********************** + +Plugins execute in a pipeline where each stage depends on the previous one: + +Discovery → Creator + The discovery plugin produces a ``PythonInfo`` object describing the source Python interpreter. This object contains + metadata about the Python version, platform, paths, and capabilities. The creator plugin receives this + ``PythonInfo`` and uses it to determine how to build the virtual environment structure. + +Creator → Seeder + The creator plugin produces a ``Creator`` object representing the newly created virtual environment. This includes + paths to the environment's ``bin`` directory, site-packages, and Python executable. The seeder plugin uses these + paths to install packages. + +Seeder → Activator + After seeding completes, activator plugins use the ``Creator`` object to generate shell activation scripts. These + scripts reference the environment's bin directory and other paths to configure the shell environment. + +This pipeline ensures that each plugin has the information it needs from previous stages. The ``PythonInfo`` flows from +discovery to creator, and the ``Creator`` object flows from creator to both seeder and activators. + +Plugin isolation +================ + +Plugins within the same extension point do not interact with each other. Only one discovery and one creator plugin can +run per invocation, though multiple activators can run simultaneously. This isolation keeps plugins simple and focused +on their specific task. diff --git a/docs/plugin/how-to.rst b/docs/plugin/how-to.rst new file mode 100644 index 000000000..ba823856e --- /dev/null +++ b/docs/plugin/how-to.rst @@ -0,0 +1,255 @@ +###################### + Plugin how-to guides +###################### + +This page provides task-oriented guides for creating each type of virtualenv plugin. + +*************************** + Create a discovery plugin +*************************** + +Discovery plugins locate Python interpreters. Register your plugin under the ``virtualenv.discovery`` entry point group. + +Implement the ``Discover`` interface: + +.. code-block:: python + + from __future__ import annotations + + from argparse import ArgumentParser + + from virtualenv.config.cli.parser import VirtualEnvOptions + from virtualenv.discovery.discover import Discover + from virtualenv.discovery.py_info import PythonInfo + + + class CustomDiscovery(Discover): + @classmethod + def add_parser_arguments(cls, parser: ArgumentParser) -> None: + parser.add_argument("--custom-opt", help="custom discovery option") + + def __init__(self, options: VirtualEnvOptions) -> None: + super().__init__(options) + self.custom_opt = options.custom_opt + + def run(self) -> PythonInfo | None: + # Locate Python interpreter and return PythonInfo + return PythonInfo.from_exe(str(self._find_python())) + + def _find_python(self) -> str: + # Implementation-specific logic + ... + +Register the entry point: + +.. code-block:: ini + + [virtualenv.discovery] + custom = your_package.discovery:CustomDiscovery + +************************* + Create a creator plugin +************************* + +Creator plugins build the virtual environment structure. Register under ``virtualenv.create``. + +Implement the ``Creator`` interface: + +.. code-block:: python + + from __future__ import annotations + + from argparse import ArgumentParser + + from virtualenv.app_data.base import AppData + from virtualenv.config.cli.parser import VirtualEnvOptions + from virtualenv.create.creator import Creator, CreatorMeta + from virtualenv.discovery.py_info import PythonInfo + + + class CustomCreator(Creator): + @classmethod + def add_parser_arguments( + cls, + parser: ArgumentParser, + interpreter: PythonInfo, + meta: CreatorMeta, + app_data: AppData, + ) -> None: + parser.add_argument("--custom-creator-opt", help="custom creator option") + + def __init__(self, options: VirtualEnvOptions, interpreter: PythonInfo) -> None: + super().__init__(options, interpreter) + self.custom_opt = options.custom_creator_opt + + def create(self) -> None: + # Create directory structure + self.bin_dir.mkdir(parents=True, exist_ok=True) + # Copy or symlink Python executable + self.install_python() + # Set up site-packages + self.install_site_packages() + # Write pyvenv.cfg + self.set_pyenv_cfg() + +Register the entry point using a naming pattern that matches platform and Python version: + +.. code-block:: ini + + [virtualenv.create] + cpython3-posix = virtualenv.create.via_global_ref.builtin.cpython.cpython3:CPython3Posix + cpython3-win = virtualenv.create.via_global_ref.builtin.cpython.cpython3:CPython3Windows + +************************ + Create a seeder plugin +************************ + +Seeder plugins install initial packages into the virtual environment. Register under ``virtualenv.seed``. + +Override ``cannot_seed`` to reject target interpreters the seeder does not support. The base returns ``None`` for every +interpreter; return a message instead and selection rejects the seeder before creating the environment, surfacing your +message to the user. A plugin can therefore serve Python versions the bundled seeders no longer ship wheels for, such as +a version past its support window. + +Implement the ``Seeder`` interface: + +.. code-block:: python + + from __future__ import annotations + + from argparse import ArgumentParser + + from virtualenv.app_data.base import AppData + from virtualenv.config.cli.parser import VirtualEnvOptions + from virtualenv.create.creator import Creator + from virtualenv.discovery.py_info import PythonInfo + from virtualenv.seed.seeder import Seeder + + + class CustomSeeder(Seeder): + @classmethod + def add_parser_arguments( + cls, parser: ArgumentParser, interpreter: PythonInfo, app_data: AppData + ) -> None: + parser.add_argument("--custom-seed-opt", help="custom seeder option") + + @classmethod + def cannot_seed(cls, interpreter: PythonInfo) -> str | None: + # ship wheels down to Python 3.6, for example + if interpreter.version_info[:2] >= (3, 6): + return None + return "custom seeder ships wheels only for Python 3.6 and later" + + def __init__(self, options: VirtualEnvOptions, enabled: bool) -> None: + super().__init__(options, enabled) + self.custom_opt = options.custom_seed_opt + + def run(self, creator: Creator) -> None: + # Install packages into creator.bin_dir / creator.script("pip") + self._install_packages(creator) + + def _install_packages(self, creator: Creator) -> None: + # Implementation-specific logic + ... + +Register the entry point: + +.. code-block:: ini + + [virtualenv.seed] + custom = your_package.seed:CustomSeeder + +**************************** + Create an activator plugin +**************************** + +Activator plugins generate shell activation scripts. Register under ``virtualenv.activate``. + +Implement the ``Activator`` interface: + +.. code-block:: python + + from __future__ import annotations + + from pathlib import Path + + from virtualenv.activation.activator import Activator + from virtualenv.create.creator import Creator + + + class CustomShellActivator(Activator): + def generate(self, creator: Creator) -> list[Path]: + # Generate activation script content + script_content = self._render_template(creator) + # Write to activation directory + dest = creator.bin_dir / self.script_name + dest.write_text(script_content) + return [dest] + + def _render_template(self, creator: Creator) -> str: + # Return activation script content + return f""" + # Custom shell activation script + export VIRTUAL_ENV="{creator.dest}" + export PATH="{creator.bin_dir}:$PATH" + """ + + @property + def script_name(self) -> str: + return "activate.custom" + +Register the entry point: + +.. code-block:: ini + + [virtualenv.activate] + bash = virtualenv.activation.bash:BashActivator + fish = virtualenv.activation.fish:FishActivator + custom = your_package.activation:CustomShellActivator + +********************************* + Package and distribute a plugin +********************************* + +Use ``pyproject.toml`` to declare entry points: + +.. code-block:: toml + + [project] + name = "virtualenv-custom-plugin" + version = "1.0.0" + dependencies = ["virtualenv>=20.0.0"] + + [project.entry-points."virtualenv.discovery"] + custom = "virtualenv_custom.discovery:CustomDiscovery" + + [project.entry-points."virtualenv.create"] + custom-posix = "virtualenv_custom.creator:CustomCreator" + + [project.entry-points."virtualenv.seed"] + custom = "virtualenv_custom.seeder:CustomSeeder" + + [project.entry-points."virtualenv.activate"] + custom = "virtualenv_custom.activator:CustomActivator" + + [build-system] + requires = ["setuptools>=61"] + build-backend = "setuptools.build_meta" + +Install your plugin alongside virtualenv: + +.. code-block:: console + + $ pip install virtualenv-custom-plugin + +Or in development mode: + +.. code-block:: console + + $ pip install -e /path/to/virtualenv-custom-plugin + +Test your plugin by creating a virtual environment: + +.. code-block:: console + + $ virtualenv --discovery=custom --creator=custom-posix --seeder=custom --activators=custom test-env diff --git a/docs/plugin/index.rst b/docs/plugin/index.rst new file mode 100644 index 000000000..79b39f952 --- /dev/null +++ b/docs/plugin/index.rst @@ -0,0 +1,39 @@ +######### + Plugins +######### + +virtualenv can be extended via plugins using Python entry points. Plugins are automatically discovered from the Python +environment where virtualenv is installed, allowing you to customize how virtual environments are created, seeded, and +activated. + +****************** + Extension points +****************** + +virtualenv provides four extension points through entry point groups: + +``virtualenv.discovery`` + Python interpreter discovery plugins. These plugins locate and identify Python interpreters that will be used as the + base for creating virtual environments. + +``virtualenv.create`` + Virtual environment creator plugins. These plugins handle the actual creation of the virtual environment structure, + including copying or symlinking the Python interpreter and standard library. + +``virtualenv.seed`` + Seed package installer plugins. These plugins install initial packages (like pip, setuptools, wheel) into newly + created virtual environments. + +``virtualenv.activate`` + Shell activation script plugins. These plugins generate shell-specific activation scripts that modify the + environment to use the virtual environment. + +All extension points follow a common pattern: virtualenv discovers registered entry points, builds CLI options from +them, and executes the selected implementations during environment creation. + +.. toctree:: + + tutorial + how-to + api + architecture diff --git a/docs/plugin/tutorial.rst b/docs/plugin/tutorial.rst new file mode 100644 index 000000000..e7f98cc61 --- /dev/null +++ b/docs/plugin/tutorial.rst @@ -0,0 +1,118 @@ +################### + Your first plugin +################### + +This tutorial walks through creating a simple discovery plugin that locates Python interpreters managed by pyenv. + +****************************** + Create the package structure +****************************** + +Set up a new Python package with the following structure: + +.. code-block:: text + + virtualenv-pyenv/ + ├── pyproject.toml + └── src/ + └── virtualenv_pyenv/ + └── __init__.py + +*************************** + Configure the entry point +*************************** + +In ``pyproject.toml``, declare your plugin as an entry point under the ``virtualenv.discovery`` group: + +.. code-block:: toml + + [project] + name = "virtualenv-pyenv" + version = "0.1.0" + dependencies = ["virtualenv>=20"] + + [project.entry-points."virtualenv.discovery"] + pyenv = "virtualenv_pyenv:PyEnvDiscovery" + + [build-system] + requires = ["setuptools>=61"] + build-backend = "setuptools.build_meta" + +********************** + Implement the plugin +********************** + +In ``src/virtualenv_pyenv/__init__.py``, implement the discovery plugin by subclassing ``Discover``: + +.. code-block:: python + + from __future__ import annotations + + import subprocess + from argparse import ArgumentParser + from pathlib import Path + + from virtualenv.config.cli.parser import VirtualEnvOptions + from virtualenv.discovery.discover import Discover + from virtualenv.discovery.py_info import PythonInfo + + + class PyEnvDiscovery(Discover): + def __init__(self, options: VirtualEnvOptions) -> None: + super().__init__(options) + self.python_spec = options.python if options.python else "python" + + @classmethod + def add_parser_arguments(cls, parser: ArgumentParser) -> None: + parser.add_argument( + "--python", + dest="python", + metavar="py", + type=str, + default=None, + help="pyenv Python version to use (e.g., 3.11.0)", + ) + + def run(self) -> PythonInfo | None: + try: + result = subprocess.run( + ["pyenv", "which", "python"], + capture_output=True, + text=True, + check=True, + ) + python_path = Path(result.stdout.strip()) + return PythonInfo.from_exe(str(python_path)) + except (subprocess.CalledProcessError, FileNotFoundError) as e: + raise RuntimeError(f"Failed to locate pyenv Python: {e}") from e + +******************** + Install the plugin +******************** + +Install your plugin in development mode alongside virtualenv: + +.. code-block:: console + + $ pip install -e virtualenv-pyenv/ + +******************* + Verify the plugin +******************* + +Check that virtualenv recognizes your plugin by running: + +.. code-block:: console + + $ virtualenv --discovery help + +The output should list ``pyenv`` as an available discovery mechanism. You can now use it: + +.. code-block:: console + + $ virtualenv --discovery=pyenv myenv + created virtual environment CPython3.11.0.final.0-64 in 234ms + creator CPython3Posix(dest=/path/to/myenv, clear=False, no_vcs_ignore=False, venv_redirect=None, global=False) + seeder FromAppData(download=False, pip=bundle, setuptools=bundle, wheel=bundle, via=copy, app_data_dir=/path) + added seed packages: pip==23.0, setuptools==65.5.0, wheel==0.38.4 + activators BashActivator,CShellActivator,FishActivator,NushellActivator,PowerShellActivator,PythonActivator diff --git a/docs/reference/api.rst b/docs/reference/api.rst new file mode 100644 index 000000000..30a5f6cf9 --- /dev/null +++ b/docs/reference/api.rst @@ -0,0 +1,40 @@ +.. _programmatic_api: + +######## + Python +######## + +The primary interface to ``virtualenv`` is the command line application. However, it can also be used programmatically +via the ``virtualenv.cli_run`` function and the ``Session`` class. + +See :doc:`../how-to/usage` for usage examples. + +******************* + virtualenv module +******************* + +.. automodule:: virtualenv + :members: + +*************** + Session class +*************** + +The ``Session`` class represents a virtualenv creation session and provides access to the created environment's +properties. + +.. currentmodule:: virtualenv.run.session + +.. autoclass:: Session + :members: + +******************* + VirtualEnvOptions +******************* + +Options namespace passed to plugin constructors, populated from the CLI, environment variables, and configuration files. + +.. currentmodule:: virtualenv.config.cli.parser + +.. autoclass:: VirtualEnvOptions + :members: diff --git a/docs/reference/cli.rst b/docs/reference/cli.rst new file mode 100644 index 000000000..bd7f889b5 --- /dev/null +++ b/docs/reference/cli.rst @@ -0,0 +1,18 @@ +############## + Command line +############## + +``virtualenv`` is primarily a command line application. All options have sensible defaults, and there is one required +argument: the name or path of the virtual environment to create. + +See :doc:`../how-to/usage` for how to select Python versions, configure defaults, and use environment variables. + +********************** + Command line options +********************** + +:command:`virtualenv [OPTIONS]` + +.. table_cli:: + :module: virtualenv.run + :func: build_parser_only diff --git a/docs/reference/compatibility.rst b/docs/reference/compatibility.rst new file mode 100644 index 000000000..22bb8dba7 --- /dev/null +++ b/docs/reference/compatibility.rst @@ -0,0 +1,98 @@ +.. _compatibility-requirements: + +############### + Compatibility +############### + +********************************** + Supported Python implementations +********************************** + +``virtualenv`` works with the following Python interpreter implementations. Only the latest patch version of each minor +version is fully supported; previous patch versions work on a best effort basis. + +CPython +======= + +``3.15 >= python_version >= 3.9`` + +PyPy +==== + +``3.11 >= python_version >= 3.9`` + +GraalPy +======= + +``24.1`` and later (Linux and macOS only). + +RustPython +========== + +Experimental support (Linux, macOS, and Windows). `RustPython `_ implements +Python 3.14. + +**************** + Support policy +**************** + +- **New versions** are added close to their release date, typically during the beta phase. +- **Old versions** are dropped 18 months after `CPython EOL `_, giving users + plenty of time to migrate. + +************************** + Version support timeline +************************** + +Major version support changes: + +- **21.5.0** (2026-06-13): dropped support for running under and creating environments for Python 3.8 and earlier. +- **20.27.0** (2024-10-17): dropped support for running under Python 3.7 and earlier. +- **20.22.0** (2023-04-19): dropped support for creating environments for Python 3.6 and earlier. +- **20.18.0** (2023-02-06): dropped support for running under Python 3.6 and earlier. + +***************************** + Supported operating systems +***************************** + +CPython is shipped in multiple forms, and each OS repackages it, often applying some customization. The platforms listed +below are tested. Unlisted platforms may work but are not explicitly supported. If you encounter issues on unlisted +platforms, please open a feature request. + +Cross-platform +============== + +These Python distributions work on Linux, macOS, and Windows: + +- Installations from `python.org `_ +- `python-build-standalone `_ builds (used by `uv + `_ and `mise `_) +- Python versions managed by `pyenv `_, `mise `_, or `asdf + `_ (shims are automatically resolved to the real binary) + +Linux +===== + +- Ubuntu 16.04 and later (both upstream and `deadsnakes `_ + builds) +- Fedora +- RHEL and CentOS +- OpenSuse +- Arch Linux + +macOS +===== + +- Python versions installed via `Homebrew `_ (works, but `not recommended + `_ -- Homebrew may upgrade or remove Python versions + without warning, breaking existing virtual environments) +- Python 3 part of XCode (Python framework builds at ``/Library/Frameworks/Python3.framework/``) + +.. note:: + + Framework builds do not support copy-based virtual environments. Use symlink or hardlink creation methods instead. + +Windows +======= + +- `Windows Store `_ Python 3.9 and later diff --git a/docs/reference/environment-layout.rst b/docs/reference/environment-layout.rst new file mode 100644 index 000000000..899070b9a --- /dev/null +++ b/docs/reference/environment-layout.rst @@ -0,0 +1,55 @@ +#################### + Environment layout +#################### + +The interpreter in a created environment answers to more than one name, so a script or tool that hard-codes any of them +keeps working. The names below are what the builtin creators write for CPython; PyPy, GraalPy and RustPython follow the +same shape under their own executable names. + +``{minor}`` stands for the target interpreter's minor version, so ``python3.{minor}`` reads as ``python3.14`` for a +CPython 3.14 target. + +******* + POSIX +******* + +``bin`` holds one real reference to the host interpreter and links the rest to it: + +==================== ====================================================================== +Name Notes +==================== ====================================================================== +``python`` References the host interpreter. +``python3`` Alias. +``python3.{minor}`` Alias carrying the target's minor version. +``python3.{minor}t`` Free-threaded builds only. +host executable name Appears when the host carries another name, such as ``pypy3.{minor}``. +==================== ====================================================================== + +********* + Windows +********* + +``Scripts`` holds copies rather than links, since symlinking the interpreter is unreliable there (`bpo-42013 +`_): + +========================= ===================================================================== +Name Notes +========================= ===================================================================== +``python.exe`` References the host interpreter. +``python3.exe`` Alias. +``python3`` Alias without the extension. +``python3.{minor}t.exe`` Free-threaded builds only. +``pythonw.exe`` Runs without opening a console window. +``pythonw3.exe`` Alias for ``pythonw.exe``. +``pythonw3.{minor}t.exe`` Free-threaded builds only. +host executable name Appears when the host carries another name, such as ``python_d.exe``. +========================= ===================================================================== + +On CPython 3.13 and later ``python.exe`` holds a copy of the ``venvlauncher.exe`` redirector from the host's standard +library rather than the interpreter binary, and ``pythonw.exe`` comes from ``venvwlauncher.exe``. The redirector reads +``pyvenv.cfg`` and hands off to the interpreter in the base prefix, which is why the environment needs no copy of the +DLLs. Its own file name stays out of ``Scripts``, matching what the standard library's ``venv`` writes. + +.. note:: + + ``--copies`` makes the POSIX layout use copies instead of links. The set of names does not change. diff --git a/docs/reference/files.rst b/docs/reference/files.rst new file mode 100644 index 000000000..e5ce3dfa8 --- /dev/null +++ b/docs/reference/files.rst @@ -0,0 +1,116 @@ +################# + Generated files +################# + +Creating an environment writes files inside the destination folder and beside it. This page lists each one, what it +holds, and how to suppress it. + +******************************* + Inside the environment folder +******************************* + +``pyvenv.cfg`` +============== + +Marks the folder as a virtual environment and points the interpreter at the Python it was built from, per `PEP 405 +`_. Deleting it breaks the environment. + +.. code-block:: ini + + home = /usr/local/python-3.14/bin + implementation = CPython + python-version = 3.14 + version_info = 3.14.6.final.0 + version = 3.14.6 + executable = /usr/local/python-3.14/bin/python3.14 + command = /usr/bin/python3 -m virtualenv /home/user/env + virtualenv = 21.7.6 + include-system-site-packages = false + base-prefix = /usr/local/python-3.14 + base-exec-prefix = /usr/local/python-3.14 + base-executable = /usr/local/python-3.14/bin/python3.14 + +Three keys carry the version, and they differ in precision and in who should read them: + +.. list-table:: + :header-rows: 1 + :widths: 25 30 45 + + - - Key + - Example + - Read it when + - - ``python-version`` + - ``3.14`` + - You want the feature release, which is what selects a wheel tag or a type-checker target. `PEP 838 + `_ defines it and every tool creating an environment must write it. + - - ``version`` + - ``3.14.6`` + - You need the patch level as well. PEP 838 discourages reading it in favor of ``python-version``. + - - ``version_info`` + - ``3.14.6.final.0`` + - You need the release level and serial. Written by virtualenv rather than by any specification. + +``prompt`` appears as an extra key when you pass ``--prompt``. The ``base-*`` keys come from the creator and vary by +creation method. + +``CACHEDIR.TAG`` +================ + +Marks the environment as regenerable cache content, following the `cache directory tagging specification +`_, so backup tools skip it. virtualenv leaves an existing file untouched. + +``.gitignore`` +============== + +Holds ``*``, keeping the environment out of Git. Skip it with ``--no-vcs-ignore``. virtualenv leaves an existing file +untouched, and writes nothing for Mercurial, Bazaar or Subversion, none of which honor ignore files in a subdirectory. + +``bin`` / ``Scripts`` +===================== + +The interpreter, the console scripts of any seeded package, and the activation scripts for each shell. +:doc:`environment-layout` lists the names the interpreter answers to, and :ref:`explanation:Activators` covers the +shells. + +******************************* + Beside the environment folder +******************************* + +``.venv`` +========= + +A :PEP:`832` redirect file holding the destination folder name, which tells editors and type checkers which environment +of the parent folder to use. virtualenv writes it in a folder holding a ``pyproject.toml`` and no ``.venv`` yet. Force +it with ``--venv-redirect``, skip it with ``--no-venv-redirect``. + +.. warning:: + + This file is provisional because PEP 832 is still a draft, and virtualenv follows the PEP as it changes. A minor or + patch release may change this file, ``--venv-redirect`` and ``--no-venv-redirect`` in backward incompatible ways. + +.. code-block:: text + + py314 + +Format rules virtualenv follows when it writes the file: + +.. list-table:: + :header-rows: 1 + :widths: 30 70 + + - - Rule + - Behavior + - - Encoding + - UTF-8 without a byte order mark. + - - Content + - One line: the destination folder name, relative to the parent folder, followed by a newline. + - - Target + - The first environment created in a folder holding a ``pyproject.toml``; with ``--venv-redirect``, the + environment created last, in any folder. + - - ``.venv`` folder + - Left alone, whether folder or symlink; virtualenv writes nothing when the destination itself is ``.venv``. + - - Existing redirect + - Kept, unless you pass ``--venv-redirect`` and its target's ``pyvenv.cfg`` carries the ``virtualenv`` key; one + that another tool wrote, or that points at a missing folder, stays either way. + - - Failure + - Logged as a warning; the environment is still created. diff --git a/docs/reference/release-artifacts.rst b/docs/reference/release-artifacts.rst new file mode 100644 index 000000000..c11355a99 --- /dev/null +++ b/docs/reference/release-artifacts.rst @@ -0,0 +1,174 @@ +################### + Release artifacts +################### + +A maintainer runs the `pre-release workflow `_, +which writes the changelog, commits the release and pushes the version tag. That tag starts the `release workflow +`_, which builds and publishes every file below. +``{version}`` stands for the release version, such as ``21.10.0``. :doc:`../how-to/verify-release` shows how to check +each file, and :ref:`release-integrity` explains what the checks prove. + +******* + Files +******* + +.. list-table:: + :header-rows: 1 + + - - File + - Where + - Contents + - - ``virtualenv-{version}-py3-none-any.whl`` + - `PyPI `_ + - the package, with its CycloneDX SBOM under ``.dist-info/sboms`` + - - ``virtualenv-{version}.tar.gz`` + - PyPI + - the source distribution + - - ``virtualenv.pyz`` + - `GitHub release `_ + - the zipapp: virtualenv and its runtime dependencies for every supported Python, with its own SBOM at the archive + root + - - ``virtualenv.pyz.intoto.jsonl`` + - GitHub release + - the Sigstore bundle holding the zipapp's SLSA provenance + - - ``virtualenv.pyz.cdx.json`` + - GitHub release + - the zipapp's CycloneDX SBOM, the same file the zipapp carries + - - ``virtualenv.cdx.json`` + - GitHub release + - the wheel's CycloneDX SBOM, the same file the wheel carries + - - ``virtualenv.spdx.json`` + - GitHub release + - the wheel's SBOM rendered as SPDX 2.3 + - - ``virtualenv.pyz`` + - ``https://bootstrap.pypa.io/virtualenv.pyz`` + - the latest release's zipapp + - - ``virtualenv.pyz`` for Python ``{x.y}`` + - ``https://bootstrap.pypa.io/virtualenv/{x.y}/virtualenv.pyz`` + - the zipapp for Python ``{x.y}`` + +The bootstrap copies come from the ``public`` directory of `pypa/get-virtualenv +`_, which the release workflow updates, and can trail the latest release. +Releases published before an asset existed do not have it; ``gh release view {version} --repo pypa/virtualenv`` lists +what a release carries. Releases from 21.11.0 on are immutable; GitHub rejects changes to their tag and assets after +publication. + +************** + Attestations +************** + +.. list-table:: + :header-rows: 1 + + - - Attestation + - Subject + - Stored at + - - PyPI publish attestation (`PEP 740 `_), predicate + ``https://docs.pypi.org/attestations/publish/v1`` + - wheel, sdist + - ``https://pypi.org/integrity/virtualenv/{version}/{file}/provenance`` + - - SLSA provenance, predicate ``https://slsa.dev/provenance/v1`` + - ``virtualenv.pyz`` + - GitHub attestations API and ``virtualenv.pyz.intoto.jsonl`` + - - CycloneDX SBOM, predicate ``https://cyclonedx.org/bom`` + - wheel, sdist + - GitHub attestations API, predicate equal to ``virtualenv.cdx.json`` + - - SPDX SBOM, predicate ``https://spdx.dev/Document/v2.3`` + - wheel, sdist + - GitHub attestations API, predicate equal to ``virtualenv.spdx.json`` + - - CycloneDX SBOM, predicate ``https://cyclonedx.org/bom`` + - ``virtualenv.pyz`` + - GitHub attestations API, predicate equal to ``virtualenv.pyz.cdx.json`` + - - GitHub release attestation, predicate ``https://in-toto.io/attestation/release/v0.2`` + - the tag's commit and every GitHub release asset, from 21.11.0 on + - GitHub attestations API, read by ``gh release verify`` + +The release workflow signs every attestation except the last through `Sigstore `_ with the +identity ``https://github.com/pypa/virtualenv/.github/workflows/release.yaml@refs/tags/{version}``. GitHub signs the +release attestation when it publishes an immutable release, with the identity ``https://dotcom.releases.github.com``. + +******* + SBOMs +******* + +Wheel SBOM +========== + +- Format: CycloneDX 1.6 JSON. +- Location in the wheel: ``virtualenv-{version}.dist-info/sboms/virtualenv.cdx.json``, following `PEP 770 + `_. Installers copy it into the installed ``.dist-info`` directory. The sdist does + not carry one. +- Components: every wheel bundled under ``virtualenv/seed/wheels/embed``, with its SHA-256, license, the packages it + vendors, and a ``virtualenv:seeded-for-python`` property per Python version that receives it; plus the runtime + dependencies declared in the wheel metadata, without versions, since the installer resolves those. +- Licenses: a component whose metadata has a ``License-Expression`` carries that SPDX expression. Otherwise it lists its + ``License`` field and ``License ::`` classifier names as declared, plus the SPDX ids they map to as concluded when + each name maps to a single SPDX license. +- Build record: the Python version and build backend packages that produced the wheel, the source commit, and the + ``SOURCE_DATE_EPOCH`` used for timestamps. Releases up to 21.10.0 also recorded the operating system, architecture and + interpreter build of the build machine. +- First release carrying it: 21.8.1. + +SPDX rendering +============== + +``virtualenv.spdx.json`` is the wheel SBOM rendered as SPDX 2.3 JSON for tools that read only SPDX. It keeps the +packages, SHA-256 hashes, declared and concluded licenses, purls and the containment, dependency and build tool +relationships. It leaves out the per-file hashes, because SPDX 2.3 requires a SHA-1 for every file and wheel ``RECORD`` +files hold SHA-256, and the build record, which SPDX 2.3 has no field for. Its document namespace ends in the CycloneDX +serial number, so both documents name the same build. + +Zipapp SBOM +=========== + +- Format: CycloneDX 1.6 JSON, at ``virtualenv.pyz.cdx.json`` in the zipapp's root and as a release asset. +- Root component: ``pkg:generic/virtualenv.pyz@{version}``. +- Components: virtualenv, with a SHA-256 per file and the embedded ``pip`` and ``setuptools`` wheels by purl and + SHA-256; and each bundled distribution, such as ``filelock`` or ``platformdirs``, with its version, license, a + ``virtualenv:loaded-for-python`` property per Python version that imports it, and a SHA-256 per file. Every file in + the archive other than the SBOM appears in it. +- Licenses: recorded the same way as in the wheel SBOM. +- Build record: the Python version and packages of the environment that built the zipapp, and the ``SOURCE_DATE_EPOCH`` + used for timestamps. Packages installed from platform-specific wheels appear without their files, which differ per + operating system and architecture. + +*************************** + Embedded wheel advisories +*************************** + +Only Python 3.9 environments receive the embedded ``pip`` 26.0.1 and ``setuptools`` 82.0.1, and both versions have +published advisories that a scanner reading the SBOMs reports. ``pip`` 26.1 and ``setuptools`` 83 and later require +Python 3.10, so ``--pip`` and ``--setuptools`` cannot pick a fixed version for Python 3.9; create environments for +Python 3.10 or newer to avoid them. + +*********************** + Build reproducibility +*********************** + +The release sets ``SOURCE_DATE_EPOCH`` to the commit time of the tag (``git log -1 --pretty=%ct``). Rebuilding the tag +with the same value reproduces the sdist byte for byte. + +The wheel, whose SBOM `hatch_build.py `_ writes, reproduces +byte for byte on any operating system and architecture when these inputs match the release: + +- the source tree, as a git checkout of the tag, since the SBOM records the commit; +- ``SOURCE_DATE_EPOCH``; +- the Python patch version, which the SBOM records; +- the versions of the build backend and its dependencies, which the SBOM lists. + +Any build frontend works, since the SBOM leaves out the installer metadata a frontend writes into the build environment. + +Wheels up to 21.10.0 recorded the build machine in their SBOM, so a rebuild of those differs in the SBOM and in +``RECORD``, which holds the SBOM's hash. + +The release pins the build backend and its dependencies by version and SHA-256 in `tasks/release-requirements.txt +`_, so a checkout of the tag carries them. + +The zipapp, built by `tasks/make_zipapp.py `_, needs +the same inputs, and its entries carry ``SOURCE_DATE_EPOCH`` as their timestamp and fixed permissions. The build takes +the distributions it bundles from `pylock.zipapp.toml +`_, a `PEP 751 `_ +lock that pins each wheel by SHA-256, and fails on a hash mismatch. A rebuild of a release from 21.11.0 on also needs +the versions of the tools that built it: the zipapp SBOM lists the packages of the build environment, and the wheel SBOM +inside the zipapp lists the backend that built that wheel. Releases after 21.13.0 also pin those tools in +``tasks/release-requirements.txt``. diff --git a/docs/render_cli.py b/docs/render_cli.py index ad77fc6a2..13bb94fda 100644 --- a/docs/render_cli.py +++ b/docs/render_cli.py @@ -74,37 +74,41 @@ def run(self): def _run_parser(self, class_n, key, name): test_name = {"creator": "can_create", "activators": "supports"} func_name = test_name.get(key) + prev = getattr(class_n, func_name) if func_name is not None else None + if func_name is not None: + setattr(class_n, func_name, self._make_patch_func(prev, key, name)) try: - if func_name is not None: - prev = getattr(class_n, func_name) - - def a(*args, **kwargs): - prev(*args, **kwargs) - if key == "activators": - return True - if key == "creator": - if name == "venv": - from virtualenv.create.via_global_ref.venv import ViaGlobalRefMeta # noqa: PLC0415 - - meta = ViaGlobalRefMeta() - meta.symlink_error = None - return meta - from virtualenv.create.via_global_ref.builtin.via_global_self_do import ( # noqa: PLC0415 - BuiltinViaGlobalRefMeta, - ) - - meta = BuiltinViaGlobalRefMeta() - meta.symlink_error = None - return meta - raise RuntimeError - - setattr(class_n, func_name, a) yield finally: if func_name is not None: - # noinspection PyUnboundLocalVariable setattr(class_n, func_name, prev) + @staticmethod + def _make_patch_func(prev, key, name): + def a(*args, **kwargs): + prev(*args, **kwargs) + if key == "activators": + return True + if key == "creator": + if name == "venv": + from virtualenv.create.via_global_ref.venv import ( # ruff:ignore[import-outside-top-level] + ViaGlobalRefMeta, + ) + + meta = ViaGlobalRefMeta() + meta.symlink_error = None + return meta + from virtualenv.create.via_global_ref.builtin.via_global_self_do import ( # ruff:ignore[import-outside-top-level] + BuiltinViaGlobalRefMeta, + ) + + meta = BuiltinViaGlobalRefMeta() + meta.symlink_error = None + return meta + raise RuntimeError + + return a + def _build_table(self, options, title, description): table = n.table() table["classes"] += ["colwidths-auto"] @@ -226,7 +230,7 @@ def register_target_option(self, target) -> None: domain.add_program_option(None, key, self.env.docname, key) -def literal_data(rawtext, app, of_type, slug, options): # noqa: ARG001 +def literal_data(rawtext, app, of_type, slug, options): # ruff:ignore[unused-function-argument] """Create a link to a BitBucket resource.""" of_class = of_type.split(".") data = getattr(__import__(".".join(of_class[:-1]), fromlist=[of_class[-1]]), of_class[-1]) diff --git a/docs/tutorial/getting-started.rst b/docs/tutorial/getting-started.rst new file mode 100644 index 000000000..12794f047 --- /dev/null +++ b/docs/tutorial/getting-started.rst @@ -0,0 +1,277 @@ +################# + Getting started +################# + +This tutorial will teach you the basics of virtualenv through hands-on practice. You'll create your first virtual +environment, install packages, and learn how to manage project dependencies. + +*************** + Prerequisites +*************** + +Before starting this tutorial, you need: + +- Python 3.9 or later installed on your system. If you use a version manager like `pyenv + `_, `mise `_, or `asdf `_, virtualenv + will automatically discover the Python version they manage. +- virtualenv installed (see :doc:`../how-to/install`). + +*************************************** + Create your first virtual environment +*************************************** + +Let's create a virtual environment called ``myproject``: + +.. code-block:: console + + $ virtualenv myproject + created virtual environment CPython3.13.2.final.0-64 in 200ms + creator CPython3Posix(dest=/home/user/myproject, clear=False, no_vcs_ignore=False, venv_redirect=None, global=False) + seeder FromAppData(download=False, pip=bundle, setuptools=bundle, via=copy, app_data_dir=/home/user/.cache/virtualenv) + activators BashActivator,CShellActivator,FishActivator,NushellActivator,PowerShellActivator,PythonActivator + +This creates a new directory called ``myproject`` containing a complete, isolated Python environment with its own copy +of Python, pip, and other tools. + +``pyvenv.cfg`` inside it records which Python the environment came from: + +.. code-block:: console + + $ grep python-version myproject/pyvenv.cfg + python-version = 3.13 + +That key tells an editor or a type checker which language version your code targets. + +The interpreter answers to several names, so a script expecting any of them keeps working inside the environment: + +.. code-block:: console + + $ ls myproject/bin/python* + myproject/bin/python myproject/bin/python3 myproject/bin/python3.13 + +When you create the environment inside a project folder, one with a ``pyproject.toml``, virtualenv writes a ``.venv`` +redirect file naming it: + +.. code-block:: console + + $ cat .venv + myproject + +Editors and type checkers read that file to find the environment, so they can offer the right interpreter before you +activate anything. A second environment in the same folder leaves the redirect as it is; pass ``--venv-redirect`` to +point it at the new one, or ``--no-venv-redirect`` to write nothing. The redirect is provisional and may change as PEP +832 evolves. :doc:`../reference/environment-layout` lists every interpreter name, and :doc:`../reference/files` covers +every file. + +************************** + Activate the environment +************************** + +To use your virtual environment, you can activate it. The activation command differs by platform: + +.. tab:: Linux/macOS + + .. code-block:: console + + $ source myproject/bin/activate + +.. tab:: Windows (PowerShell) + + .. code-block:: console + + PS> .\myproject\Scripts\Activate.ps1 + +.. tab:: Windows (CMD) + + .. code-block:: console + + C:\> .\myproject\Scripts\activate.bat + +After activation, your prompt changes to show the active environment: + +.. code-block:: console + + (myproject) $ + +You can verify that Python is now running from inside the virtual environment: + +.. tab:: Linux/macOS + + .. code-block:: console + + (myproject) $ which python + /home/user/myproject/bin/python + +.. tab:: Windows (PowerShell) + + .. code-block:: console + + (myproject) PS> where.exe python + C:\Users\user\myproject\Scripts\python.exe + +.. tab:: Windows (CMD) + + .. code-block:: console + + (myproject) C:\> where.exe python + C:\Users\user\myproject\Scripts\python.exe + +******************* + Install a package +******************* + +With the environment activated, install a package using pip: + +.. code-block:: console + + (myproject) $ pip install requests + Collecting requests + Using cached requests-2.32.3-py3-none-any.whl (64 kB) + Installing collected packages: requests + Successfully installed requests-2.32.3 + +Verify that the package is installed only inside your virtual environment: + +.. code-block:: console + + (myproject) $ python -c "import requests; print(requests.__file__)" + /home/user/myproject/lib/python3.13/site-packages/requests/__init__.py + +The path shows that ``requests`` is installed in the virtual environment, not in your system Python. + +************ + Deactivate +************ + +When you're done working in the virtual environment, deactivate it: + +.. code-block:: console + + (myproject) $ deactivate + $ + +The prompt returns to normal, and Python commands now use your system Python again. + +************************ + Use without activation +************************ + +Activation is a convenience, not a requirement. You can run any executable from the virtual environment directly by +using its full path: + +.. tab:: Linux/macOS + + .. code-block:: console + + $ myproject/bin/python -c "import sys; print(sys.prefix)" + /home/user/myproject + + $ myproject/bin/pip install httpx + +.. tab:: Windows (PowerShell) + + .. code-block:: console + + PS> .\myproject\Scripts\python.exe -c "import sys; print(sys.prefix)" + C:\Users\user\myproject + + PS> .\myproject\Scripts\pip.exe install httpx + +.. tab:: Windows (CMD) + + .. code-block:: console + + C:\> .\myproject\Scripts\python.exe -c "import sys; print(sys.prefix)" + C:\Users\user\myproject + + C:\> .\myproject\Scripts\pip.exe install httpx + +This is especially useful in scripts, CI pipelines, and automation where modifying the shell environment is unnecessary. + +*********************** + Set up a real project +*********************** + +Now let's apply what you've learned to a real project workflow: + +.. code-block:: console + + $ mkdir myapp && cd myapp + $ virtualenv venv + $ source venv/bin/activate # or use the appropriate command for your platform + (venv) $ pip install flask requests + (venv) $ pip freeze > requirements.txt + +The ``requirements.txt`` file now contains your project's dependencies: + +.. code-block:: text + + blinker==1.9.0 + certifi==2025.1.31 + charset-normalizer==3.4.1 + click==8.1.8 + flask==3.1.0 + idna==3.10 + itsdangerous==2.2.0 + Jinja2==3.1.5 + MarkupSafe==3.0.2 + requests==2.32.3 + urllib3==2.3.0 + werkzeug==3.1.3 + +This file lets you recreate the exact environment later. Let's test this: + +.. code-block:: console + + (venv) $ deactivate + $ rm -rf venv + $ virtualenv venv + $ source venv/bin/activate + (venv) $ pip install -r requirements.txt + +All packages are reinstalled exactly as before. Here's the complete workflow: + +.. mermaid:: + + graph TD + A[Create virtual environment] --> B[Activate] + B --> C[Install packages] + C --> D[Freeze to requirements.txt] + D --> E[Deactivate & clean up] + E --> F[Recreate virtual environment] + F --> G[Install from requirements.txt] + G --> H[Ready to work] + + style A fill:#2563eb,stroke:#1d4ed8,color:#fff + style B fill:#6366f1,stroke:#4f46e5,color:#fff + style C fill:#6366f1,stroke:#4f46e5,color:#fff + style D fill:#6366f1,stroke:#4f46e5,color:#fff + style E fill:#d97706,stroke:#b45309,color:#fff + style F fill:#6366f1,stroke:#4f46e5,color:#fff + style G fill:#6366f1,stroke:#4f46e5,color:#fff + style H fill:#16a34a,stroke:#15803d,color:#fff + +****************** + What you learned +****************** + +In this tutorial, you learned how to: + +- Create a virtual environment with ``virtualenv``. +- Activate and deactivate virtual environments on different platforms. +- Install packages in isolation from your system Python. +- Save project dependencies with ``pip freeze``. +- Reproduce environments using ``requirements.txt``. +- Let editors find your environment through the ``.venv`` redirect file. + +************ + Next steps +************ + +Now that you understand the basics, explore these topics: + +- :doc:`../how-to/usage` for selecting specific Python versions, configuring defaults, and advanced usage patterns. +- :doc:`../explanation` for understanding how virtualenv works under the hood and how it compares to ``venv``. +- :doc:`../reference/cli` for all available command line options and flags. +- :doc:`../how-to/verify-release` to check that the virtualenv you downloaded came from its release workflow. +- :doc:`../reference/files` for every file virtualenv writes inside and beside an environment. diff --git a/docs/user_guide.rst b/docs/user_guide.rst deleted file mode 100644 index 82cc235b1..000000000 --- a/docs/user_guide.rst +++ /dev/null @@ -1,311 +0,0 @@ -User Guide -========== - -Quick start ------------ -Create the environment (creates a folder in your current directory) - .. code-block:: console - - virtualenv env_name -In Linux or Mac, activate the new python environment - .. code-block:: console - - source env_name/bin/activate -Or in Windows - .. code-block:: console - - .\env_name\Scripts\activate -Confirm that the env is successfully selected - .. code-block:: console - - which python3 - - -Introduction ------------- - -Virtualenv has one basic command: - -.. code-block:: console - - virtualenv venv - -.. note:: - - When creating a virtual environment, it's recommended to use a specific Python version, for example, by invoking - virtualenv with ``python3.10 -m virtualenv venv``. If you use a generic command like ``python3 -m virtualenv venv``, - the created environment will be linked to ``/usr/bin/python3``. This can be problematic because when a new Python - version is installed on the system, the ``/usr/bin/python3`` symlink will likely be updated to point to the new - version. This will cause the virtual environment to inadvertently use the new Python version, which is often not the - desired behavior. Using a specific version ensures that the virtual environment is tied to that exact version, - providing stability and predictability. - -This will create a python virtual environment of the same version as virtualenv, installed into the subdirectory -``venv``. The command line tool has quite a few of flags that modify the tool's behavior, for a -full list make sure to check out :ref:`cli_flags`. - -The tool works in two phases: - -- **Phase 1** discovers a python interpreter to create a virtual environment from (by default this is the same python - as the one ``virtualenv`` is running from, however we can change this via the :option:`p` option). -- **Phase 2** creates a virtual environment at the specified destination (:option:`dest`), this can be broken down into - four further sub-steps: - - - create a python that matches the target python interpreter from phase 1, - - install (bootstrap) seed packages (one or more of :pypi:`pip`, :pypi:`setuptools`, :pypi:`wheel`) in the created - virtual environment, - - install activation scripts into the binary directory of the virtual environment (these will allow end users to - *activate* the virtual environment from various shells). - - create files that mark the virtual environment as to be ignored by version control systems (currently we support - Git only, as Mercurial, Bazaar or SVN do not support ignore files in subdirectories). This step can be skipped - with the :option:`no-vcs-ignore` option. - - -The python in your new virtualenv is effectively isolated from the python that was used to create it. - -Python discovery ----------------- - -The first thing we need to be able to create a virtual environment is a python interpreter. This will describe to the -tool what type of virtual environment you would like to create, think of it as: version, architecture, implementation. - -``virtualenv`` being a python application has always at least one such available, the one ``virtualenv`` itself is -using, and as such this is the default discovered element. This means that if you install ``virtualenv`` under -python ``3.8``, virtualenv will by default create virtual environments that are also of version ``3.8``. - -Created python virtual environments are usually not self-contained. A complete python packaging is usually made up of -thousands of files, so it's not efficient to install the entire python again into a new folder. Instead virtual -environments are mere shells, that contain little within themselves, and borrow most from the system python (this is what -you installed, when you installed python itself). This does mean that if you upgrade your system python your virtual -environments *might* break, so watch out. The upside of this, referring to the system python, is that creating virtual -environments can be fast. - -Here we'll describe the built-in mechanism (note this can be extended though by plugins). The CLI flag :option:`p` or -:option:`python` allows you to specify a python specifier for what type of virtual environment you would like, the -format is either: - -- a relative/absolute path to a Python interpreter, - -- a specifier identifying the Python implementation, version, architecture in the following format: - - .. code-block:: - - {python implementation name}{version}{architecture} - - We have the following restrictions: - - - the python implementation is all alphabetic characters (``python`` means any implementation, and if is missing it - defaults to ``python``), - - the version is a dot separated version number optionally followed by ``t`` for free-threading, - - the architecture is either ``-64`` or ``-32`` (missing means ``any``). - - For example: - - - ``python3.8.1`` means any python implementation having the version ``3.8.1``, - - ``3`` means any python implementation having the major version ``3``, - - ``3.13t`` means any python implementation having the version ``3.13`` with free threading, - - ``cpython3`` means a ``CPython`` implementation having the version ``3``, - - ``pypy2`` means a python interpreter with the ``PyPy`` implementation and major version ``2``. - - Given the specifier ``virtualenv`` will apply the following strategy to discover/find the system executable: - - - If we're on Windows look into the Windows registry, and check if we see any registered Python implementations that - match the specification. This is in line with expectation laid out inside - `PEP-514 `_ - - If `uv-managed `_ Python installations are available, use the - first one that matches the specification. - - Try to discover a matching python executable within the folders enumerated on the ``PATH`` environment variable. - In this case we'll try to find an executable that has a name roughly similar to the specification (for exact logic, - please see the implementation code). - -.. warning:: - - As detailed above, virtual environments usually just borrow things from the system Python, they don't actually contain - all the data from the system Python. The version of the python executable is hardcoded within the python exe itself. - Therefore, if you upgrade your system Python, your virtual environment will still report the version before the - upgrade, even though now other than the executable all additional content (standard library, binary libs, etc) are - of the new version. - - Barring any major incompatibilities (rarely the case) the virtual environment will continue working, but other than - the content embedded within the python executable it will behave like the upgraded version. If such a virtual - environment python is specified as the target python interpreter, we will create virtual environments that match the - new system Python version, not the version reported by the virtual environment. - -Creators --------- - -These are what actually setup the virtual environment, usually as a reference against the system python. virtualenv -at the moment has two types of virtual environments: - -- ``venv`` - this delegates the creation process towards the ``venv`` module, as described in - `PEP 405 `_. This is only available on Python interpreters having version - ``3.5`` or later, and also has the downside that virtualenv **must** create a process to invoke that module (unless - virtualenv is installed in the system python), which can be an expensive operation (especially true on Windows). - -- ``builtin`` - this means ``virtualenv`` is able to do the creation operation itself (by knowing exactly what files to - create and what system files need to be referenced). The creator with name ``builtin`` is an alias on the first - creator that's of this type (we provide creators for various target environments, that all differ in actual create - operations, such as CPython 2 on Windows, PyPy2 on Windows, CPython3 on Posix, PyPy3 on Posix, and so on; for a full - list see :option:`creator`). - -Seeders -------- -These will install for you some seed packages (one or more of: :pypi:`pip`, :pypi:`setuptools`, :pypi:`wheel`) that -enables you to install additional python packages into the created virtual environment (by invoking pip). Installing -:pypi:`setuptools` is disabled by default on Python 3.12+ environments. :pypi:`wheel` is only installed on Python 3.8, by default. There are two main seed mechanisms available: - -- ``pip`` - this method uses the bundled pip with virtualenv to install the seed packages (note, a new child process - needs to be created to do this, which can be expensive especially on Windows). -- ``app-data`` - this method uses the user application data directory to create install images. These images are needed - to be created only once, and subsequent virtual environments can just link/copy those images into their pure python - library path (the ``site-packages`` folder). This allows all but the first virtual environment creation to be blazing - fast (a ``pip`` mechanism takes usually 98% of the virtualenv creation time, so by creating this install image that - we can just link into the virtual environments install directory we can achieve speedups of shaving the initial - 1 minute and 10 seconds down to just 8 seconds in case of a copy, or ``0.8`` seconds in case symlinks are available - - this is on Windows, Linux/macOS with symlinks this can be as low as ``100ms`` from 3+ seconds). - To override the filesystem location of the seed cache, one can use the - ``VIRTUALENV_OVERRIDE_APP_DATA`` environment variable. - -.. _wheels: - -Wheels -~~~~~~ - -To install a seed package via either ``pip`` or ``app-data`` method virtualenv needs to acquire a wheel of the target -package. These wheels may be acquired from multiple locations as follows: - -- ``virtualenv`` ships out of box with a set of embed ``wheels`` for all three seed packages (:pypi:`pip`, - :pypi:`setuptools`, :pypi:`wheel`). These are packaged together with the virtualenv source files, and only change upon - upgrading virtualenv. Different Python versions require different versions of these, and because virtualenv supports a - wide range of Python versions, the number of embedded wheels out of box is greater than 3. Whenever newer versions of - these embedded packages are released upstream ``virtualenv`` project upgrades them, and does a new release. Therefore, - upgrading virtualenv periodically will also upgrade the version of the seed packages. -- However, end users might not be able to upgrade virtualenv at the same speed as we do new releases. Therefore, a user - might request to upgrade the list of embedded wheels by invoking virtualenv with the :option:`upgrade-embed-wheels` - flag. If the operation is triggered in such a manual way subsequent runs of virtualenv will always use the upgraded - embed wheels. - - The operation can trigger automatically too, as a background process upon invocation of virtualenv, if no such upgrade - has been performed in the last 14 days. It will only start using automatically upgraded wheel if they have been - released for more than 28 days, and the automatic upgrade finished at least an hour ago: - - - the 28 days period should guarantee end users are not pulling in automatically releases that have known bugs within, - - the one hour period after the automatic upgrade finished is implemented so that continuous integration services do - not start using a new embedded versions half way through. - - - The automatic behavior might be disabled via the :option:`no-periodic-update` configuration flag/option. To acquire - the release date of a package virtualenv will perform the following: - - - lookup ``https://pypi.org/pypi//json`` (primary truth source), - - save the date the version was first discovered, and wait until 28 days passed. -- Users can specify a set of local paths containing additional wheels by using the :option:`extra-search-dir` command - line argument flag. - -When searching for a wheel to use virtualenv performs lookup in the following order: - -- embedded wheels, -- upgraded embedded wheels, -- extra search dir. - -Bundled wheels are all three above together. If neither of the locations contain the requested wheel version or -:option:`download` option is set will use ``pip`` download to load the latest version available from the index server. - -.. _distribution_wheels: - -Embed wheels for distributions -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Custom distributions often want to use their own set of wheel versions to distribute instead of the one virtualenv -releases on PyPi. The reason for this is trying to keep the system versions of those packages in sync with what -virtualenv uses. In such cases they should patch the module `virtualenv.seed.wheels.embed -`_, making sure to provide the function -``get_embed_wheel`` (which returns the wheel to use given a distribution/python version). The ``BUNDLE_FOLDER``, -``BUNDLE_SUPPORT`` and ``MAX`` variables are needed if they want to use virtualenv's test suite to validate. - -Furthermore, they might want to disable the periodic update by patching the -`virtualenv.seed.embed.base_embed.PERIODIC_UPDATE_ON_BY_DEFAULT -`_ -to ``False``, and letting the system update mechanism to handle this. Note in this case the user might still request an -upgrade of the embedded wheels by invoking virtualenv via :option:`upgrade-embed-wheels`, but no longer happens -automatically, and will not alter the OS provided wheels. - -Activators ----------- -These are activation scripts that will mangle with your shell's settings to ensure that commands from within the python -virtual environment take priority over your system paths. For example, if invoking ``pip`` from your shell returned the -system python's pip before activation, once you do the activation this should refer to the virtual environments ``pip``. -Note, though that all we do is change priority; so, if your virtual environments ``bin``/``Scripts`` folder does not -contain some executable, this will still resolve to the same executable it would have resolved before the activation. - -For a list of shells we provide activators see :option:`activators`. The location of these is right alongside the Python -executables: usually ``Scripts`` folder on Windows, ``bin`` on POSIX. They are called ``activate``, plus an -extension that's specific per activator, with no extension for Bash. You can invoke them, usually by source-ing them. -The source command might vary by shell - e.g. on Bash it’s ``source`` (or ``.``): - -.. code-block:: console - - source venv/bin/activate - -The activate script prepends the virtual environment’s binary folder onto the ``PATH`` environment variable. It’s -really just convenience for doing so, since you could do the same yourself. - -Note that you don't have to activate a virtual environment to use it. You can instead use the full paths to its -executables, rather than relying on your shell to resolve them to your virtual environment. - -Activator scripts also modify your shell prompt to indicate which environment is currently active, by prepending the -environment name (or the name specified by ``--prompt`` when initially creating the environment) in brackets, like -``(venv)``. You can disable this behavior by setting the environment variable ``VIRTUAL_ENV_DISABLE_PROMPT`` to any -value. You can also get the environment name via the environment variable ``VIRTUAL_ENV_PROMPT`` if you want to -customize your prompt, for example. - -The scripts also provision a ``deactivate`` command that will allow you to undo the operation: - -.. code-block:: console - - deactivate - -.. note:: - - If using Powershell, the ``activate`` script is subject to the - `execution policies `_ on the system. By default, Windows - 7 and later, the system's execution policy is set to ``Restricted``, meaning no scripts like the ``activate`` script - are allowed to be executed. - - However, that can't stop us from changing that slightly to allow it to be executed. You may relax the system - execution policy to allow running of local scripts without verifying the code signature using the following: - - .. code-block:: powershell - - Set-ExecutionPolicy RemoteSigned - - Since the ``activate.ps1`` script is generated locally for each virtualenv, it is not considered a remote script and - can then be executed. - -A longer explanation of this can be found within Allison Kaptur's 2013 blog post: `There's no magic: virtualenv -edition `_ explains how virtualenv uses bash and -Python and ``PATH`` and ``PYTHONHOME`` to isolate virtual environments' paths. - -.. _programmatic_api: - -Programmatic API ----------------- - -At the moment ``virtualenv`` offers only CLI level interface. If you want to trigger invocation of Python environments -from within Python you should be using the ``virtualenv.cli_run`` method; this takes an ``args`` argument where you can -pass the options the same way you would from the command line. The run will return a session object containing data -about the created virtual environment. - -.. code-block:: python - - from virtualenv import cli_run - - cli_run(["venv"]) - -.. automodule:: virtualenv - :members: - -.. currentmodule:: virtualenv.run.session - -.. autoclass:: Session - :members: diff --git a/hatch_build.py b/hatch_build.py new file mode 100644 index 000000000..df70edbbf --- /dev/null +++ b/hatch_build.py @@ -0,0 +1,641 @@ +from __future__ import annotations + +import ast +import base64 +import csv +import hashlib +import json +import platform +import re +import shutil +import subprocess +import sys +import tempfile +import uuid +import zipfile +from datetime import datetime, timezone +from email.parser import Parser +from email.utils import getaddresses +from importlib.metadata import distributions +from io import StringIO +from itertools import starmap +from pathlib import Path +from typing import TYPE_CHECKING, Any, Final + +from hatchling.builders.hooks.plugin.interface import BuildHookInterface +from hatchling.builders.utils import get_reproducible_timestamp +from packaging.licenses import InvalidLicenseExpression, canonicalize_license_expression +from packaging.requirements import Requirement + +if TYPE_CHECKING: + from email.message import Message + from importlib.metadata import Distribution, PackageMetadata + from typing import NotRequired, TypedDict + + from hatchling.metadata.core import CoreMetadata + from packaging.specifiers import Specifier + + class _OrganizationalEntity(TypedDict): + name: str + url: list[str] + + class _Hash(TypedDict): + alg: str + content: str + + class Property(TypedDict): + name: str + value: str + + class _ExternalReference(TypedDict): + type: str + url: str + comment: NotRequired[str] + + class _LicenseName(TypedDict): + name: str + acknowledgement: str + + class _LicenseId(TypedDict): + id: str + acknowledgement: str + + class _NamedLicense(TypedDict): + license: _LicenseName | _LicenseId + + class _LicenseExpression(TypedDict): + expression: str + acknowledgement: str + + _License = _NamedLicense | _LicenseExpression + + class _Method(TypedDict): + technique: str + confidence: int + value: str + + class _Identity(TypedDict): + field: str + confidence: int + methods: list[_Method] + + class _Evidence(TypedDict): + identity: list[_Identity] + + Component = TypedDict( + "Component", + { + "type": str, + "bom-ref": str, + "name": str, + "version": NotRequired[str], + "purl": NotRequired[str], + "supplier": NotRequired[_OrganizationalEntity], + "authors": NotRequired[list[dict[str, str]]], + "description": NotRequired[str], + "licenses": NotRequired[list[_License]], + "copyright": NotRequired[str], + "hashes": NotRequired[list[_Hash]], + "externalReferences": NotRequired[list[_ExternalReference]], + "properties": NotRequired[list[Property]], + "evidence": NotRequired[_Evidence], + "components": NotRequired[list["Component"]], + }, + ) + + class _Dependency(TypedDict): + ref: str + dependsOn: list[str] + + class _Ref(TypedDict): + ref: str + + class _Output(TypedDict): + type: str + resource: _Ref + + class _Input(TypedDict): + environmentVars: list[Property] + + _Workflow = TypedDict( + "_Workflow", + { + "bom-ref": str, + "uid": str, + "name": str, + "taskTypes": list[str], + "resourceReferences": list[_Ref], + "inputs": NotRequired[list[_Input]], + "outputs": list[_Output], + }, + ) + + _Formula = TypedDict("_Formula", {"bom-ref": str, "workflows": list[_Workflow]}) + + class _Lifecycle(TypedDict): + phase: str + + class _Tools(TypedDict): + components: list[Component] + + class _Metadata(TypedDict): + timestamp: str + lifecycles: list[_Lifecycle] + tools: _Tools + manufacturer: _OrganizationalEntity + authors: NotRequired[list[dict[str, str]]] + supplier: _OrganizationalEntity + component: Component + licenses: NotRequired[list[_License]] + properties: NotRequired[list[Property]] + + class _Composition(TypedDict): + aggregate: str + dependencies: list[str] + + class Body(TypedDict): + metadata: _Metadata + components: list[Component] + dependencies: list[_Dependency] + compositions: NotRequired[list[_Composition]] + formulation: list[_Formula] + + _Header = TypedDict( + "_Header", {"$schema": str, "bomFormat": str, "specVersion": str, "serialNumber": str, "version": int} + ) + + class Document(_Header, Body): + pass + + +_ROOT: Final[Path] = Path(__file__).resolve().parent +_EMBED: Final[Path] = _ROOT / "src" / "virtualenv" / "seed" / "wheels" / "embed" +_REPOSITORY: Final[str] = "https://github.com/pypa/virtualenv" +SBOM_NAMESPACE: Final[uuid.UUID] = uuid.uuid5(uuid.NAMESPACE_URL, f"{_REPOSITORY}/sboms") +PYPA: Final[_OrganizationalEntity] = {"name": "Python Packaging Authority", "url": ["https://www.pypa.io"]} +# the label normalization and mapping cyclonedx-py applies to Project-URL entries, plus "source", so the same +# metadata yields the same reference types whichever generator produced the document +_URL_LABEL_TO_REFERENCE_TYPE: Final[dict[str, str]] = { + "bugtracker": "issue-tracker", + "issuetracker": "issue-tracker", + "issues": "issue-tracker", + "bugreports": "issue-tracker", + "tracker": "issue-tracker", + "home": "website", + "homepage": "website", + "download": "distribution", + "documentation": "documentation", + "docs": "documentation", + "changelog": "release-notes", + "changes": "release-notes", + "source": "vcs", + "repository": "vcs", + "github": "vcs", + "chat": "chat", +} +# core metadata headers copied verbatim onto a component as properties +_METADATA_PROPERTIES: Final[tuple[str, ...]] = ( + "Requires-Python", + "Requires-Dist", + "Provides-Extra", + "Classifier", + "Keywords", +) +# only the license classifiers that name exactly one SPDX license; "BSD License" or "Apache Software License" leave the +# version or variant open +_CLASSIFIER_LICENSES: Final[dict[str, str]] = { + "ISC License (ISCL)": "ISC", + "MIT License": "MIT", + "MIT No Attribution License (MIT-0)": "MIT-0", + "Mozilla Public License 2.0 (MPL 2.0)": "MPL-2.0", + "Python Software Foundation License": "PSF-2.0", + "The Unlicense (Unlicense)": "Unlicense", +} +_RECORD_HASH_ALGORITHMS: Final[dict[str, str]] = { + "md5": "MD5", + "sha1": "SHA-1", + "sha256": "SHA-256", + "sha384": "SHA-384", + "sha512": "SHA-512", +} + + +class SbomBuildHook(BuildHookInterface): + """Write a PEP 770 CycloneDX SBOM into the wheel's ``.dist-info/sboms/`` directory. + + General-purpose SBOM scanners (syft, cyclonedx-py, GitHub's dependency graph) read declared dependency metadata or + an installed environment, and virtualenv's bundled pip and setuptools wheels are neither: they are data files + embedded under ``src/virtualenv/seed/wheels/embed/``, invisible to every one of those tools. Each bundled wheel is + described from its own ``METADATA`` and ``RECORD`` and hashed from its bytes, so a wheel bump needs no separate SBOM + update. + + The document also records the build toolchain (the Python version, every distribution in the isolated build env with + the files of the pure-Python ones, and the dependency graph between them), which PEP 770 calls out as what a third + party needs to verify build reproducibility, plus the source revision when known. Nothing about the build machine + goes in, so a rebuild with the same toolchain on another OS or architecture produces the same wheel; release + attestations identify the CI run and the builder instead. + + """ + + PLUGIN_NAME = "sbom" + + def initialize(self, version: str, build_data: dict[str, Any]) -> None: # ruff:ignore[unused-method-argument] + # `version` here is hatchling's build variant name (e.g. "standard"), not the package version + if self.target_name != "wheel" or "sbom_files" not in build_data: + # sbom_files only exists on hatchling>=1.28, which build-system.requires excludes for the + # Python versions old enough to still need hatchling<1.28 for building at all + return + document = _cyclonedx_document(self.metadata.core, self.metadata.version) + # written outside self.directory, which hatchling also uses as the final wheel output location, so a + # stray copy here would sit next to the built artifacts and trip `twine check dist/*` + out = Path(tempfile.mkdtemp(prefix="virtualenv-sbom-")) / "virtualenv.cdx.json" + out.write_text(json.dumps(document, indent=2) + "\n", encoding="utf-8") + build_data["sbom_files"].append(str(out)) + + +def _cyclonedx_document(core: CoreMetadata, version: str) -> Document: + root = _root_component(core, version) + bundled = [_bundled_component(wheel) for wheel in sorted(_EMBED.glob("*.whl"))] + declared = [_declared_dependency(requirement) for requirement in core.dependencies] + tools, tool_dependencies = build_tools(version) + bundled_dependencies: Final[list[_Dependency]] = [ + {"ref": component["bom-ref"], "dependsOn": references} + for component in bundled + if (references := [child["bom-ref"] for child in component["components"] if child["type"] == "library"]) + ] + body: Final[Body] = { + "metadata": { + "timestamp": timestamp(), + "lifecycles": [{"phase": "build"}], + "tools": {"components": tools}, + "manufacturer": PYPA, + "authors": _contacts(core.maintainers_data["name"], core.maintainers_data["email"]), + "supplier": PYPA, + "component": root, + "licenses": [{"expression": core.license_expression, "acknowledgement": "declared"}], + "properties": [ + { + "name": "virtualenv:sbom:scope", + "value": "Components are the wheels bundled as data files under src/virtualenv/seed/wheels/embed" + " and the direct runtime dependencies declared in the wheel's core metadata. The latter carry" + " no version because they are resolved at install time, not at build time.", + }, + ], + }, + "components": [*bundled, *declared], + "dependencies": [ + {"ref": root["bom-ref"], "dependsOn": [component["bom-ref"] for component in [*bundled, *declared]]}, + *bundled_dependencies, + *tool_dependencies, + ], + # transitive runtime dependencies are unknown until install time + "compositions": [{"aggregate": "incomplete", "dependencies": [root["bom-ref"]]}], + "formulation": [{"bom-ref": "formula:wheel-build", "workflows": [_workflow(root, tools)]}], + } + return { + "$schema": "http://cyclonedx.org/schema/bom-1.6.schema.json", + "bomFormat": "CycloneDX", + "specVersion": "1.6", + # derived from the content so identical documents share a serial and any difference gets a new one + "serialNumber": f"urn:uuid:{uuid.uuid5(SBOM_NAMESPACE, json.dumps(body, sort_keys=True))}", + "version": 1, + **body, + } + + +def _root_component(core: CoreMetadata, version: str) -> Component: + purl = _purl(core.name, version) + wheel_name = f"{core.name}-{version}-py3-none-any.whl" + license_lines = (_ROOT / "LICENSE").read_text(encoding="utf-8").splitlines() + references = list(starmap(_external_reference, core.urls.items())) + references += [ + {"type": "distribution", "url": f"https://pypi.org/project/{core.name}/{version}/"}, + {"type": "attestation", "url": f"https://pypi.org/integrity/{core.name}/{version}/{wheel_name}/provenance"}, + {"type": "security-contact", "url": f"{_REPOSITORY}/security/policy"}, + {"type": "advisories", "url": f"{_REPOSITORY}/security/advisories"}, + {"type": "license", "url": f"{_REPOSITORY}/blob/main/LICENSE"}, + ] + properties: Final[list[Property]] = [{"name": "virtualenv:requires-python", "value": core.requires_python}] + properties.extend({"name": "python:classifier", "value": classifier} for classifier in core.classifiers) + properties.extend({"name": "python:keyword", "value": keyword} for keyword in core.keywords) + if commit := _commit(): + references.append({"type": "vcs", "url": f"{_REPOSITORY}/tree/{commit}", "comment": "exact source revision"}) + properties.append({"name": "virtualenv:vcs-commit", "value": commit}) + return { + "type": "application", + "bom-ref": purl, + "supplier": PYPA, + "authors": _contacts(core.maintainers_data["name"], core.maintainers_data["email"]), + "name": core.name, + "version": version, + "description": core.description, + "licenses": [{"expression": core.license_expression, "acknowledgement": "declared"}], + "copyright": next(line for line in license_lines if line.startswith("Copyright")), + "purl": purl, + "externalReferences": references, + "properties": properties, + } + + +def _purl(name: str, version: str | None = None) -> str: + normalized = re.sub(r"[-_.]+", "-", name).lower() + return f"pkg:pypi/{normalized}@{version}" if version else f"pkg:pypi/{normalized}" + + +def _external_reference(label: str, url: str) -> _ExternalReference: + reference_type = _URL_LABEL_TO_REFERENCE_TYPE.get(re.sub(r"[^a-z]", "", label.lower()), "other") + return {"type": reference_type, "url": url, "comment": f"Project-URL: {label}"} + + +def _commit() -> str | None: + if not (_ROOT / ".git").exists() or (git := shutil.which("git")) is None: # building from an sdist + return None + return subprocess.run( + [git, "rev-parse", "HEAD"], check=True, capture_output=True, text=True, cwd=_ROOT + ).stdout.strip() + + +def _contacts(names: list[str], addresses: list[str]) -> list[dict[str, str]]: + contacts = [{"name": name} for name in names] + for name, email in getaddresses(addresses): + contacts.append({key: value for key, value in (("name", name), ("email", email)) if value}) + return contacts + + +def _bundled_component(wheel: Path) -> Component: + with zipfile.ZipFile(wheel) as archive: + members: Final[list[str]] = sorted(archive.namelist()) + metadata_name: Final[str] = next( + name for name in members if name.count("/") == 1 and name.endswith(".dist-info/METADATA") + ) + metadata = Parser().parsestr(archive.read(metadata_name).decode("utf-8")) + record = archive.read(metadata_name.replace("METADATA", "RECORD")).decode("utf-8") + vendored_metadata: Final[list[tuple[str, Message]]] = [] + for name in members: + if not name.endswith("/_vendor/vendor.txt"): + continue + for line in archive.read(name).decode("utf-8").splitlines(): + if not (requirement := line.partition("#")[0].strip()): + continue + parsed: Final[Requirement] = Requirement(requirement) + pins: Final[list[Specifier]] = list(parsed.specifier) + if len(pins) != 1 or pins[0].operator != "==" or "*" in pins[0].version: + msg: Final[str] = f"Unpinned vendored dependency in {wheel.name}:{name}: {requirement}" + raise ValueError(msg) + vendored_metadata.append(( + name, + Parser().parsestr(f"Name: {parsed.name}\nVersion: {pins[0].version}\n"), + )) + vendored_metadata.extend( + (name, Parser().parsestr(archive.read(name).decode("utf-8"))) + for name in members + if "/_vendor/" in name and name.endswith(".dist-info/METADATA") + ) + component = component_from_metadata(metadata, "library") + if any(reference["url"].startswith("https://github.com/pypa/") for reference in component["externalReferences"]): + component["supplier"] = PYPA + sha256 = hashlib.sha256(wheel.read_bytes()).hexdigest() + component["hashes"] = [{"alg": "SHA-256", "content": sha256}] + component["externalReferences"].append( + {"type": "distribution", "url": f"https://pypi.org/project/{metadata['Name']}/{metadata['Version']}/"}, + ) + seeded_for: Final[list[Property]] = [ + {"name": "virtualenv:seeded-for-python", "value": python_version} + for python_version, wheels in _bundle_support().items() + if wheel.name in wheels.values() + ] + component["properties"] = [ + {"name": "virtualenv:bundled-wheel", "value": wheel.relative_to(_ROOT).as_posix()}, + *seeded_for, + *component["properties"], + ] + component["evidence"] = { + "identity": [ + { + "field": "purl", + "confidence": 1, + "methods": [{"technique": "manifest-analysis", "confidence": 1, "value": metadata_name}], + }, + { + "field": "hash", + "confidence": 1, + "methods": [{"technique": "hash-comparison", "confidence": 1, "value": sha256}], + }, + ], + } + component["components"] = [ + _file_component(component["bom-ref"], path, digest, size) + for path, digest, size in (row for row in csv.reader(StringIO(record, newline="")) if row) + if digest + ] + vendored_components: Final[dict[str, Component]] = {} + for source, vendored in vendored_metadata: + child: Final[Component] = component_from_metadata(vendored, "library") + child["bom-ref"] = f"{component['bom-ref']}#vendored/{child['purl']}" + child["properties"].append({"name": "virtualenv:vendored-manifest", "value": source}) + child["evidence"] = { + "identity": [ + { + "field": "purl", + "confidence": 1, + "methods": [{"technique": "manifest-analysis", "confidence": 1, "value": source}], + } + ], + } + vendored_components[child["purl"]] = child + component["components"].extend(vendored_components[key] for key in sorted(vendored_components)) + return component + + +def component_from_metadata(metadata: PackageMetadata, component_type: str) -> Component: + name, version = metadata["Name"], metadata["Version"] + purl = _purl(name, version) + component: Final[Component] = { + "type": component_type, + "bom-ref": purl, + "name": name, + "version": version, + "purl": purl, + } + if summary := metadata.get("Summary"): + component["description"] = summary + if authors := _contacts( + metadata.get_all("Author", []) + metadata.get_all("Maintainer", []), + metadata.get_all("Author-email", []) + metadata.get_all("Maintainer-email", []), + ): + component["authors"] = authors + if licenses := _licenses(metadata): + component["licenses"] = licenses + component["externalReferences"] = [ + _external_reference(label.strip(), url.strip()) + for label, url in (entry.split(",", 1) for entry in metadata.get_all("Project-URL", [])) + ] + if home_page := metadata.get("Home-page"): + component["externalReferences"].insert(0, {"type": "website", "url": home_page}) + component["properties"] = [ + {"name": f"python:{header.lower()}", "value": value} + for header in _METADATA_PROPERTIES + for value in metadata.get_all(header, []) + ] + return component + + +def _licenses(metadata: PackageMetadata) -> list[_License]: + if expression := metadata.get("License-Expression"): + return [{"expression": expression, "acknowledgement": "declared"}] + names = [ + entry.rsplit(" :: ", 1)[-1] for entry in metadata.get_all("Classifier", []) if entry.startswith("License :: ") + ] + if (declared := metadata.get("License")) and "\n" not in declared: + names.append(declared) + licenses: Final[list[_License]] = [{"license": {"name": name, "acknowledgement": "declared"}} for name in names] + # tasks/license_policy.py judges SPDX ids only, so a component keeps its bare names when any one of them does not + # resolve and the check reports it instead of judging a partial set + identifiers: Final[list[str]] = [identifier for name in names if (identifier := _spdx_license(name)) is not None] + if len(identifiers) == len(names): + licenses.extend( + {"license": {"id": identifier, "acknowledgement": "concluded"}} for identifier in dict.fromkeys(identifiers) + ) + return licenses + + +def _spdx_license(name: str) -> str | None: + if identifier := _CLASSIFIER_LICENSES.get(name): + return identifier + try: + canonical = canonicalize_license_expression(name) + except InvalidLicenseExpression: + return None + # a CycloneDX license id holds one entry of the SPDX license list, never an expression or a LicenseRef + return canonical if re.fullmatch(r"[A-Za-z0-9.-]+", canonical) and not canonical.startswith("LicenseRef-") else None + + +def _bundle_support() -> dict[str, dict[str, str]]: + tree = ast.parse((_EMBED / "__init__.py").read_text(encoding="utf-8")) + for node in tree.body: + if isinstance(node, ast.Assign) and any( + isinstance(target, ast.Name) and target.id == "BUNDLE_SUPPORT" for target in node.targets + ): + return ast.literal_eval(node.value) + msg = f"BUNDLE_SUPPORT not found in {_EMBED / '__init__.py'}" + raise RuntimeError(msg) + + +def _file_component(parent_ref: str, path: str, digest: str, size: str) -> Component: + # RECORD stores "=", CycloneDX wants lowercase hex + algorithm, _, encoded = digest.partition("=") + raw = base64.urlsafe_b64decode(encoded + "=" * (-len(encoded) % 4)) + return { + "type": "file", + "bom-ref": f"{parent_ref}#{path}", + "name": path, + "hashes": [{"alg": _RECORD_HASH_ALGORITHMS[algorithm], "content": raw.hex()}], + "properties": [{"name": "size", "value": size}], + } + + +def _declared_dependency(requirement: str) -> Component: + parsed = Requirement(requirement) + component: Final[Component] = { + "type": "library", + # the same distribution can be declared more than once with different markers, so the purl alone is not unique + "bom-ref": f"requires-dist:{requirement}", + "name": parsed.name, + "purl": _purl(parsed.name), + "externalReferences": [{"type": "distribution", "url": f"https://pypi.org/project/{parsed.name}/"}], + "properties": [{"name": "virtualenv:requires-dist", "value": requirement}], + } + if parsed.marker is not None: + component["properties"].append({"name": "virtualenv:environment-marker", "value": str(parsed.marker)}) + return component + + +def build_tools(package_version: str) -> tuple[list[Component], list[_Dependency]]: + hook = Path(__file__) + interpreter = f"pkg:generic/{sys.implementation.name}@{platform.python_version()}" + tools: Final[list[Component]] = [ + { + "type": "application", + "bom-ref": "tool:hatch_build.py", + "name": hook.name, + "version": package_version, + "hashes": [{"alg": "SHA-256", "content": hashlib.sha256(hook.read_bytes()).hexdigest()}], + "externalReferences": [{"type": "vcs", "url": f"{_REPOSITORY}/blob/main/hatch_build.py"}], + }, + { + "type": "platform", + "bom-ref": f"tool:{interpreter}", + "name": sys.implementation.name, + "version": platform.python_version(), + "purl": interpreter, + "properties": [{"name": "python:implementation", "value": platform.python_implementation()}], + }, + ] + # bom-refs are prefixed because the same distribution can be both a build tool and a bundled component + installed = {_purl(distribution.metadata["Name"]): distribution for distribution in distributions()} + tool_dependencies: Final[list[_Dependency]] = [] + for distribution in (installed[key] for key in sorted(installed)): + component = component_from_metadata(distribution.metadata, "library") + component["bom-ref"] = f"tool:{component['purl']}" + # a platform wheel installs files built for the build machine's OS and architecture, so listing them would + # tie the document to that machine + if Parser().parsestr(distribution.read_text("WHEEL") or "")["Root-Is-Purelib"] == "true": + component["components"] = [ + _file_component( + component["bom-ref"], file.as_posix(), f"{file.hash.mode}={file.hash.value}", str(file.size) + ) + for file in distribution.files or [] + # console-script launchers live outside site-packages and embed the build env's interpreter path in + # their shebang, so their hash differs on every build and says nothing about the distribution; the + # installer metadata names the build frontend that set up the env rather than the distribution + if file.hash is not None + and not file.as_posix().startswith("../") + and not ( + file.parent.suffix == ".dist-info" and file.name in {"INSTALLER", "REQUESTED", "direct_url.json"} + ) + ] + tools.append(component) + tool_dependencies.append({"ref": component["bom-ref"], "dependsOn": _depends_on(distribution, installed)}) + return tools, tool_dependencies + + +def _depends_on(distribution: Distribution, installed: dict[str, Distribution]) -> list[str]: + refs = set() + for requirement in map(Requirement, distribution.requires or []): + if (requirement.marker is None or requirement.marker.evaluate()) and ( + dependency := installed.get(_purl(requirement.name)) + ): + refs.add(f"tool:{_purl(dependency.metadata['Name'], dependency.version)}") + return sorted(refs) + + +def _workflow(root: Component, tools: list[Component]) -> _Workflow: + workflow: Final[_Workflow] = { + "bom-ref": "workflow:wheel-build", + "uid": "wheel-build", + "name": "build the wheel and this SBOM", + "taskTypes": ["build"], + "resourceReferences": [{"ref": tool["bom-ref"]} for tool in tools], + "outputs": [{"type": "artifact", "resource": {"ref": root["bom-ref"]}}], + } + workflow["inputs"] = [ + {"environmentVars": [{"name": "SOURCE_DATE_EPOCH", "value": str(get_reproducible_timestamp())}]}, + ] + return workflow + + +def timestamp() -> str: + return datetime.fromtimestamp(get_reproducible_timestamp(), tz=timezone.utc).isoformat() + + +__all__ = [ + "PYPA", + "SBOM_NAMESPACE", + "Body", + "Component", + "Document", + "Property", + "build_tools", + "component_from_metadata", + "timestamp", +] diff --git a/pylock.zipapp.toml b/pylock.zipapp.toml new file mode 100644 index 000000000..7ad75a2e3 --- /dev/null +++ b/pylock.zipapp.toml @@ -0,0 +1,58 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile pyproject.toml --universal --python-version 3.9 -o pylock.zipapp.toml +lock-version = "1.0" +created-by = "uv" +requires-python = ">=3.9" + +[[packages]] +name = "distlib" +version = "0.4.3" +sdist = { url = "https://files.pythonhosted.org/packages/c9/02/bd72be9134d25ed783ecbbc38a539ffaefbf90c78418c7fb7229600dbac7/distlib-0.4.3.tar.gz", upload-time = 2026-06-12T08:04:52Z, size = 615141, hashes = { sha256 = "f152097224a0ae24be5a0f6bae1b9359af82133bce63f98a95f86cae1aede9ed" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/02/08/9c41fb51ab5b43eb21674aff13df270e8ba6c4b29c8624e328dc7a9482af/distlib-0.4.3-py2.py3-none-any.whl", upload-time = 2026-06-12T08:04:50Z, size = 470628, hashes = { sha256 = "4b0ce306c966eb73bc3a7b6abad017c556dadd92c44701562cd528ac7fde4d5b" } }] + +[[packages]] +name = "filelock" +version = "3.19.1" +marker = "python_full_version < '3.10'" +sdist = { url = "https://files.pythonhosted.org/packages/40/bb/0ab3e58d22305b6f5440629d20683af28959bf793d98d11950e305c1c326/filelock-3.19.1.tar.gz", upload-time = 2025-08-14T16:56:03Z, size = 17687, hashes = { sha256 = "66eda1888b0171c998b35be2bcc0f6d75c388a7ce20c3f3f37aa8e96c2dddf58" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/42/14/42b2651a2f46b022ccd948bca9f2d5af0fd8929c4eec235b8d6d844fbe67/filelock-3.19.1-py3-none-any.whl", upload-time = 2025-08-14T16:56:01Z, size = 15988, hashes = { sha256 = "d38e30481def20772f5baf097c122c3babc4fcdb7e14e57049eb9d88c6dc017d" } }] + +[[packages]] +name = "filelock" +version = "4.0.9" +marker = "python_full_version >= '3.10'" +sdist = { url = "https://files.pythonhosted.org/packages/70/51/2bc9e529f154fad99b6cd0073e609291eb32fd23581b32362d33d164d316/filelock-4.0.9.tar.gz", upload-time = 2026-10-01T23:09:14Z, size = 251200, hashes = { sha256 = "635e7d67fa92654eed444e75e9ca18426d34e77ad9c469bf4373f75a932f7b22" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/01/4f/83454fafd628e1e7e1726d74e44fb2332be5969d04c6182ca1fecb6c580e/filelock-4.0.9-py3-none-any.whl", upload-time = 2026-10-01T23:09:12Z, size = 110368, hashes = { sha256 = "9287fd61b99a806e5202be29a83034c0808a1b9830e820537c2f5773981f7eeb" } }] + +[[packages]] +name = "packaging" +version = "26.3" +sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", upload-time = 2026-08-04T18:15:28Z, size = 313412, hashes = { sha256 = "94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", upload-time = 2026-08-04T18:15:27Z, size = 129956, hashes = { sha256 = "d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c" } }] + +[[packages]] +name = "platformdirs" +version = "4.4.0" +marker = "python_full_version < '3.10'" +sdist = { url = "https://files.pythonhosted.org/packages/23/e8/21db9c9987b0e728855bd57bff6984f67952bea55d6f75e055c46b5383e8/platformdirs-4.4.0.tar.gz", upload-time = 2025-08-26T14:32:04Z, size = 21634, hashes = { sha256 = "ca753cf4d81dc309bc67b0ea38fd15dc97bc30ce419a7f58d13eb3bf14c4febf" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/40/4b/2028861e724d3bd36227adfa20d3fd24c3fc6d52032f4a93c133be5d17ce/platformdirs-4.4.0-py3-none-any.whl", upload-time = 2025-08-26T14:32:02Z, size = 18654, hashes = { sha256 = "abd01743f24e5287cd7a5db3752faf1a2d65353f38ec26d98e25a6db65958c85" } }] + +[[packages]] +name = "platformdirs" +version = "4.12.2" +marker = "python_full_version >= '3.10'" +sdist = { url = "https://files.pythonhosted.org/packages/17/c8/721b3855fe457da514fe249247d404b9b39c5d16532278f70ebaa6acf18b/platformdirs-4.12.2.tar.gz", upload-time = 2026-09-29T18:27:56Z, size = 59353, hashes = { sha256 = "eab5f70271a490ef74618bb314fbb86e3c7e82fa3b9c922c2ea0e0a1a155d329" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/d0/89/446044f33aba0348d35e433f56d12206d010a5281a1df54054d4cfb82388/platformdirs-4.12.2-py3-none-any.whl", upload-time = 2026-09-29T18:27:54Z, size = 32457, hashes = { sha256 = "29dbf06d96c500bc6bdbce75fb0a14d63279c93b1842f97e72a135b33e856983" } }] + +[[packages]] +name = "python-discovery" +version = "1.6.1" +sdist = { url = "https://files.pythonhosted.org/packages/0c/57/250bd238b966cece44328235eb85290045d059265fdaf7527a3a958123db/python_discovery-1.6.1.tar.gz", upload-time = 2026-09-18T01:31:53Z, size = 84338, hashes = { sha256 = "cf87d3627dfb4412437fdd5b13eae402607722998d21567993aedbc59b23c15e" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/16/7d/e9ffbadfbf89c93848412d04594135c4ae8c1d37d9e053b9c3ed718fabc4/python_discovery-1.6.1-py3-none-any.whl", upload-time = 2026-09-18T01:31:52Z, size = 38664, hashes = { sha256 = "d43fcdef879fe795352bd13ccf8d185ba5a9f86f36cfcd00529f596e737442b3" } }] + +[[packages]] +name = "typing-extensions" +version = "4.16.0" +marker = "python_full_version < '3.11'" +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", upload-time = 2026-07-02T08:40:05Z, size = 113555, hashes = { sha256 = "dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", upload-time = 2026-07-02T08:40:04Z, size = 45571, hashes = { sha256 = "481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8" } }] diff --git a/pyproject.toml b/pyproject.toml index 85ff87ae8..c01da2327 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,8 +1,9 @@ [build-system] build-backend = "hatchling.build" requires = [ - "hatch-vcs>=0.3", - "hatchling>=1.17.1", + "hatch-vcs>=0.5", + "hatchling<1.28,>=1.27; python_version<'3.10'", + "hatchling>=1.28; python_version>='3.10'", ] [project] @@ -15,10 +16,15 @@ keywords = [ "virtual", ] license = "MIT" +license-files = [ + "LICENSE", + "THIRD-PARTY-NOTICES.md", +] maintainers = [ { name = "Bernat Gabor", email = "gaborjbernat@gmail.com" }, + { name = "Rahul Devikar", email = "rahuldevikar5512@gmail.com" }, ] -requires-python = ">=3.8" +requires-python = ">=3.9" classifiers = [ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", @@ -27,13 +33,13 @@ classifiers = [ "Operating System :: Microsoft :: Windows", "Operating System :: POSIX", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: 3.15", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Topic :: Software Development :: Libraries", @@ -44,38 +50,21 @@ dynamic = [ "version", ] dependencies = [ - "distlib>=0.3.7,<1", - "filelock>=3.12.2,<4", - "importlib-metadata>=6.6; python_version<'3.8'", - "platformdirs>=3.9.1,<5", - "typing-extensions>=4.13.2; python_version<'3.11'", -] -optional-dependencies.docs = [ - "furo>=2023.7.26", - "proselint>=0.13", - "sphinx>=7.1.2,!=7.3", - "sphinx-argparse>=0.4", - "sphinxcontrib-towncrier>=0.2.1a0", - "towncrier>=23.6", -] -optional-dependencies.test = [ - "covdefaults>=2.3", - "coverage>=7.2.7", - "coverage-enable-subprocess>=1", - "flaky>=3.7", - "packaging>=23.1", - "pytest>=7.4", - "pytest-env>=0.8.2", - "pytest-freezer>=0.4.8; platform_python_implementation=='PyPy' or platform_python_implementation=='GraalVM' or (platform_python_implementation=='CPython' and sys_platform=='win32' and python_version>='3.13')", - "pytest-mock>=3.11.1", - "pytest-randomly>=3.12", - "pytest-timeout>=2.1", - "setuptools>=68", - "time-machine>=2.10; platform_python_implementation=='CPython'", + "distlib<1,>=0.4.3", + "filelock<=3.19.1,>=3.19.1; python_version<'3.10'", + "filelock>=3.24.2,<5; python_version>='3.10'", + "packaging>=26.3", + "platformdirs<5,>=4.4", + "python-discovery>=1.6.1", + "typing-extensions>=4.16; python_version<'3.11'", ] +urls."thanks.dev" = "https://thanks.dev/u/gh/gaborbernat" +urls.Changelog = "https://virtualenv.pypa.io/en/latest/changelog.html" urls.Documentation = "https://virtualenv.pypa.io" +urls.Funding = "https://github.com/sponsors/gaborbernat" urls.Homepage = "https://github.com/pypa/virtualenv" urls.Source = "https://github.com/pypa/virtualenv" +urls.Tidelift = "https://tidelift.com/funding/github/pypi/virtualenv" urls.Tracker = "https://github.com/pypa/virtualenv/issues" scripts.virtualenv = "virtualenv.__main__:run_with_catch" entry-points."virtualenv.activate".bash = "virtualenv.activation.bash:BashActivator" @@ -85,28 +74,145 @@ entry-points."virtualenv.activate".fish = "virtualenv.activation.fish:FishActiva entry-points."virtualenv.activate".nushell = "virtualenv.activation.nushell:NushellActivator" entry-points."virtualenv.activate".powershell = "virtualenv.activation.powershell:PowerShellActivator" entry-points."virtualenv.activate".python = "virtualenv.activation.python:PythonActivator" -entry-points."virtualenv.create".cpython3-mac-brew = "virtualenv.create.via_global_ref.builtin.cpython.mac_os:CPython3macOsBrew" -entry-points."virtualenv.create".cpython3-mac-framework = "virtualenv.create.via_global_ref.builtin.cpython.mac_os:CPython3macOsFramework" -entry-points."virtualenv.create".cpython3-posix = "virtualenv.create.via_global_ref.builtin.cpython.cpython3:CPython3Posix" -entry-points."virtualenv.create".cpython3-win = "virtualenv.create.via_global_ref.builtin.cpython.cpython3:CPython3Windows" +entry-points."virtualenv.activate".xonsh = "virtualenv.activation.xonsh:XonshActivator" +entry-points."virtualenv.create".cpython3-mac-brew = """\ + virtualenv.create.via_global_ref.builtin.cpython.mac_os:CPython3macOsBrew\ + """ +entry-points."virtualenv.create".cpython3-mac-framework = """\ + virtualenv.create.via_global_ref.builtin.cpython.mac_os:CPython3macOsFramework\ + """ +entry-points."virtualenv.create".cpython3-posix = """\ + virtualenv.create.via_global_ref.builtin.cpython.cpython3:CPython3Posix\ + """ +entry-points."virtualenv.create".cpython3-win = """\ + virtualenv.create.via_global_ref.builtin.cpython.cpython3:CPython3Windows\ + """ entry-points."virtualenv.create".graalpy-posix = "virtualenv.create.via_global_ref.builtin.graalpy:GraalPyPosix" entry-points."virtualenv.create".graalpy-win = "virtualenv.create.via_global_ref.builtin.graalpy:GraalPyWindows" entry-points."virtualenv.create".pypy3-posix = "virtualenv.create.via_global_ref.builtin.pypy.pypy3:PyPy3Posix" entry-points."virtualenv.create".pypy3-win = "virtualenv.create.via_global_ref.builtin.pypy.pypy3:Pypy3Windows" +entry-points."virtualenv.create".rustpython-posix = """\ + virtualenv.create.via_global_ref.builtin.rustpython:RustPythonPosix\ + """ +entry-points."virtualenv.create".rustpython-win = """\ + virtualenv.create.via_global_ref.builtin.rustpython:RustPythonWindows\ + """ entry-points."virtualenv.create".venv = "virtualenv.create.via_global_ref.venv:Venv" entry-points."virtualenv.discovery".builtin = "virtualenv.discovery.builtin:Builtin" entry-points."virtualenv.seed".app-data = "virtualenv.seed.embed.via_app_data.via_app_data:FromAppData" entry-points."virtualenv.seed".pip = "virtualenv.seed.embed.pip_invoke:PipInvoke" +[dependency-groups] +dev = [ + { include-group = "docs" }, + { include-group = "lint" }, + { include-group = "pkg-meta" }, + { include-group = "tasks" }, + { include-group = "type" }, +] +test = [ + "coverage-enable-subprocess>=1", + { include-group = "coverage" }, + "packaging>=26.3", + "pytest>=8.4.2", + "pytest-env>=1.1.5", + """\ + pytest-freezer>=0.4.8; platform_python_implementation=='PyPy' or platform_python_implementation=='GraalVM' or \ + platform_python_implementation=='RustPython' or (platform_python_implementation=='CPython' and sys_platform=='win32' \ + and python_version>='3.13')\ + """, + "pytest-mock>=3.15.1", + "pytest-randomly>=4.0.1", + "pytest-rerunfailures>=16.0.1", + "pytest-timeout>=2.4", + "pytest-xdist>=3.8", + "setuptools>=82.0.1", + "time-machine>=2.19; platform_python_implementation=='CPython'", + "tomli>=2.4.1; python_version<'3.11'", +] +type = [ + "ty>=0.0.84", + { include-group = "test" }, +] +docs = [ + "furo>=2025.12.19", + "pre-commit-uv>=4.2", + "sphinx!=7.3,>=7.4.7", + "sphinx-argparse>=0.4", + "sphinx-autodoc-typehints>=3.6.2", + "sphinx-copybutton>=0.5.2", + "sphinx-inline-tabs>=2025.12.21.14", + "sphinx-llm!=1.1,>=1", + "sphinxcontrib-mermaid>=2", + "sphinxcontrib-towncrier>=0.2.1a0", + "towncrier>=26.9", + "uv>=0.12.22", +] +coverage = [ + "covdefaults>=2.3", + "coverage>=7.10.7", +] +fuzz = [ + # atheris only ships prebuilt Linux wheels for these CPython versions; everywhere else it needs a + # custom clang+libFuzzer build, so skip it there rather than triggering a doomed source build. + "atheris>=3.1; sys_platform=='linux' and python_version>='3.12' and python_version<'3.15'", +] +lint = [ + "pre-commit-uv>=4.2", +] +pkg-meta = [ + "check-sdist>=1.6", + "check-wheel-contents>=0.6.3", + "twine>=6.2", + "uv>=0.12.22", + { include-group = "sbom" }, +] +property = [ + "hypothesis>=6.141.1", + { include-group = "test" }, +] +release = [ + "docstrfmt>=2.2.1", + "gitpython>=3.2", + "hatch-vcs>=0.5", + "hatchling>=1.28", + "packaging>=26.3", + "pypi-attestations>=0.0.28", + "towncrier>=26.9", + "tox>=4.45", + "tox-uv>=1.28.1", + "uv>=0.12.22", + { include-group = "sbom" }, +] +sbom = [ + # The stable release still imports a removed typing API: https://github.com/beartype/beartype/issues/618 + "beartype>=0.23.0rc2; python_version>='3.15'", + "cyclonedx-python-lib[json-validation]>=11.12", + "spdx-tools>=0.8.5", +] +tasks = [ + "docstrfmt>=2.2.1", + "gitpython>=3.2", + "hatchling>=1.28", + "pre-commit-uv>=4.2", + "ruff>=0.16.10", + "towncrier>=26.9", + { include-group = "test" }, +] + [tool.hatch] +version.source = "vcs" build.hooks.vcs.version-file = "src/virtualenv/version.py" +build.hooks.custom.path = "hatch_build.py" build.targets.sdist.include = [ + "/.pre-commit-config.yaml", + "/pylock.zipapp.toml", "/src", - "/tests", "/tasks", - "/tox.ini", + "/tests", + "/tox.toml", + "/hatch_build.py", ] -version.source = "vcs" [tool.ruff] line-length = 120 @@ -117,36 +223,49 @@ lint.select = [ "ALL", ] lint.ignore = [ - "ANN", # no type checking added yet - "COM812", # conflict with formatter - "CPY", # No copyright header - "D10", # no docstrings - "D40", # no imperative mode for docstrings - "D203", # `one-blank-line-before-class` (D203) and `no-blank-line-before-class` (D211) are incompatible - "D212", # `multi-line-summary-first-line` (D212) and `multi-line-summary-second-line` (D213) are incompatible - "DOC", # no restructuredtext support - "INP001", # ignore implicit namespace packages - "ISC001", # conflict with formatter - "PLR0914", # Too many local variables - "PLR0917", # Too many positional arguments - "PLR6301", # Method could be a function, class method, or static method - "PLW1510", # no need for check for subprocess - "PTH", # no pathlib, <=39 has problems on Windows with absolute/resolve, can revisit once we no longer need 39 - "S104", # Possible binding to all interfaces - "S404", # Using subprocess is alright - "S603", # subprocess calls are fine + "boolean-type-hint-positional-argument", # boolean positional args - pre-existing signatures, changing would break public API + "CPY", # No copyright header + "D10", # no docstrings + "D40", # no imperative mode for docstrings + "DOC", # no restructuredtext support + "escape-sequence-in-docstring", # use r if any backslashes - conflicts with docstrfmt + "hardcoded-bind-all-interfaces", # Possible binding to all interfaces + "implicit-namespace-package", # ignore implicit namespace packages + "incorrect-blank-line-before-class", # `one-blank-line-before-class` (D203) and `no-blank-line-before-class` (D211) are incompatible + "line-too-long", # line too long - handled by ruff format and docstrfmt + "missing-blank-line-after-summary", # 1 blank line required between summary and description - conflicts with docstrfmt + "missing-trailing-comma", # conflict with formatter + "multi-line-summary-first-line", # `multi-line-summary-first-line` (D212) and `multi-line-summary-second-line` (D213) are incompatible + "multi-line-summary-second-line", # multi-line-summary-second-line - conflicts with docstrfmt + "new-line-after-last-paragraph", # multi-line docstring closing quotes - conflicts with docstrfmt + "no-self-use", # Method could be a function, class method, or static method + "non-empty-init-module", # `__init__` module should only contain docstrings and re-exports + "PTH", # no pathlib, <=39 has problems on Windows with absolute/resolve, can revisit once we no longer need 39 + "single-line-implicit-string-concatenation", # conflict with formatter + "subprocess-run-without-check", # no need for check for subprocess + "subprocess-without-shell-equals-true", # subprocess calls are fine + "suspicious-subprocess-import", # Using subprocess is alright + "too-many-locals", # Too many local variables + "too-many-positional-arguments", # Too many positional arguments + "unnecessary-multiline-docstring", # fits on one line - conflicts with docstrfmt +] +lint.per-file-ignores."docs/**/*.py" = [ + "ANN", # don't require type annotations in docs scripts ] lint.per-file-ignores."src/virtualenv/activation/python/activate_this.py" = [ - "F821", # ignore undefined template string placeholders + "undefined-name", # ignore undefined template string placeholders ] lint.per-file-ignores."tests/**/*.py" = [ - "D", # don't care about documentation in tests - "FBT", # don't care about booleans as positional arguments in tests - "INP001", # no implicit namespace - "PLC2701", # Private import - "PLR2004", # Magic value used in comparison, consider replacing with a constant variable - "S101", # asserts allowed in tests - "S603", # `subprocess` call: check for execution of untrusted input + "ANN", # don't require type annotations in tests (pytest fixtures make this impractical) + "assert", # asserts allowed in tests + "D", # don't care about documentation in tests + "FBT", # don't care about booleans as positional arguments in tests + "implicit-namespace-package", # no implicit namespace + "import-private-name", # Private import + "magic-value-comparison", # Magic value used in comparison, consider replacing with a constant variable + "pytest-fixture-autouse", # shared setup fixtures are meant to apply to every test + "start-process-with-partial-path", # partial executable path is fine in tests + "subprocess-without-shell-equals-true", # `subprocess` call: check for execution of untrusted input ] lint.isort = { known-first-party = [ "virtualenv", @@ -157,51 +276,85 @@ lint.preview = true [tool.codespell] builtin = "clear,usage,en-GB_to_en-US" +# mitre: the CWE catalog lives at cwe.mitre.org, which codespell would rewrite to "miter" +ignore-words-list = "acknowledgement,intoto,mitre" count = true +[tool.check-sdist] +default-ignore = false +git-only = [ + ".dockerignore", + ".gitattributes", + ".github/", + ".markdownlint.yaml", + ".oss-scanner/", + ".readthedocs.yml", + "_typos.toml", + "docs/", + "scorecard.yml", +] +sdist-only = [ "src/virtualenv/version.py" ] + [tool.pyproject-fmt] -max_supported_python = "3.14" +max_supported_python = "3.15" -[tool.pytest.ini_options] -markers = [ +[tool.ty] +rules.unused-ignore-comment = "ignore" # some ignores are platform/version-specific (e.g., ctypes.windll on Linux) + +[tool.pytest] +ini_options.addopts = "--showlocals -m 'not property'" +ini_options.markers = [ + "graalpy: minimal test suite for GraalPy validation", + "property: hypothesis property tests, run by their own tox environment", "slow", ] -timeout = 600 -addopts = "--showlocals --no-success-flaky-report" -env = [ +ini_options.env = [ "PYTHONIOENCODING=utf-8", ] +ini_options.timeout = 120 [tool.coverage] -html.show_contexts = true -html.skip_covered = false -report.omit = [ - # site.py is run before the coverage can be enabled, no way to measure coverage on this - "**/src/virtualenv/create/via_global_ref/builtin/python2/site.py", - "**/src/virtualenv/create/via_global_ref/_virtualenv.py", - "**/src/virtualenv/activation/python/activate_this.py", - "**/src/virtualenv/seed/wheels/embed/pip-*.whl/pip/**", -] -paths.source = [ - "src", - "**/site-packages", -] -report.fail_under = 76 run.source = [ "${_COVERAGE_SRC}", "tests", ] run.dynamic_context = "test_function" run.parallel = true +run.relative_files = true run.plugins = [ "covdefaults", ] -run.relative_files = true +paths.source = [ + "src", + "**/site-packages", +] +report.fail_under = 76 +report.omit = [ + "**/src/virtualenv/activation/python/activate_this.py", + "**/src/virtualenv/create/via_global_ref/_virtualenv.py", + # site.py is run before the coverage can be enabled, no way to measure coverage on this + "**/src/virtualenv/create/via_global_ref/builtin/python2/site.py", + "**/src/virtualenv/seed/wheels/embed/pip-*.whl/pip/**", + # hatch-vcs writes version.py at build time, so the checkout that combines every run has no source for it + "**/virtualenv/version.py", +] +html.show_contexts = true +html.skip_covered = false [tool.towncrier] -name = "tox" -filename = "docs/changelog.rst" +name = "virtualenv" directory = "docs/changelog" +filename = "docs/changelog.rst" +template = "docs/changelog/template.jinja2" title_format = false issue_format = ":issue:`{issue}`" -template = "docs/changelog/template.jinja2" +underlines = [ "*", "=", "-" ] +type = [ + # listing the types explicitly drops towncrier's "misc" default and makes a misspelt suffix fail + # instead of being silently skipped, as .bufix.rst and .docs.rst were + { directory = "feature", name = "Features", showcontent = true }, + { directory = "bugfix", name = "Bugfixes", showcontent = true }, + { directory = "doc", name = "Improved Documentation", showcontent = true }, + { directory = "deprecation", name = "Deprecations", showcontent = true }, + { directory = "removal", name = "Removals", showcontent = true } +] diff --git a/scorecard.yml b/scorecard.yml new file mode 100644 index 000000000..ca7d7abf7 --- /dev/null +++ b/scorecard.yml @@ -0,0 +1,5 @@ +annotations: + - checks: + - binary-artifacts + reasons: + - reason: remediated # embedded pip/setuptools wheels are pinned and sha256-verified at build time diff --git a/src/virtualenv/__main__.py b/src/virtualenv/__main__.py index 49f59da38..767e93ea2 100644 --- a/src/virtualenv/__main__.py +++ b/src/virtualenv/__main__.py @@ -5,29 +5,38 @@ import os import sys from timeit import default_timer +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import MutableMapping + + from virtualenv.config.cli.parser import VirtualEnvOptions + from virtualenv.run.session import Session LOGGER = logging.getLogger(__name__) -def run(args=None, options=None, env=None): +def run( + args: list[str] | None = None, options: VirtualEnvOptions | None = None, env: MutableMapping[str, str] | None = None +) -> None: env = os.environ if env is None else env start = default_timer() - from virtualenv.run import cli_run # noqa: PLC0415 - from virtualenv.util.error import ProcessCallFailedError # noqa: PLC0415 + from virtualenv.run import cli_run # ruff:ignore[import-outside-top-level] + from virtualenv.util.error import ProcessCallFailedError # ruff:ignore[import-outside-top-level] if args is None: args = sys.argv[1:] try: - session = cli_run(args, options, env) + session = cli_run(args, options, env=env) LOGGER.warning(LogSession(session, start)) except ProcessCallFailedError as exception: - print(f"subprocess call failed for {exception.cmd} with code {exception.code}") # noqa: T201 - print(exception.out, file=sys.stdout, end="") # noqa: T201 - print(exception.err, file=sys.stderr, end="") # noqa: T201 - raise SystemExit(exception.code) # noqa: B904 + print(f"subprocess call failed for {exception.cmd} with code {exception.code}") # ruff:ignore[print] + print(exception.out, file=sys.stdout, end="") # ruff:ignore[print] + print(exception.err, file=sys.stderr, end="") # ruff:ignore[print] + raise SystemExit(exception.code) # ruff:ignore[raise-without-from-inside-except] except OSError as exception: if exception.errno == errno.EMFILE: - print( # noqa: T201 + print( # ruff:ignore[print] "OSError: [Errno 24] Too many open files. You may need to increase your OS open files limit.\n" " On macOS/Linux, try 'ulimit -n 2048'.\n" " For Windows, this is not a common issue, but you can try to close some applications.", @@ -37,7 +46,7 @@ def run(args=None, options=None, env=None): class LogSession: - def __init__(self, session, start) -> None: + def __init__(self, session: Session, start: float) -> None: self.session = session self.start = start @@ -59,25 +68,28 @@ def __str__(self) -> str: return "\n".join(lines) -def run_with_catch(args=None, env=None): - from virtualenv.config.cli.parser import VirtualEnvOptions # noqa: PLC0415 +def run_with_catch(args: list[str] | None = None, env: MutableMapping[str, str] | None = None) -> None: + from virtualenv.config.cli.parser import VirtualEnvOptions # ruff:ignore[import-outside-top-level] env = os.environ if env is None else env options = VirtualEnvOptions() try: run(args, options, env) - except (KeyboardInterrupt, SystemExit, Exception) as exception: # noqa: BLE001 + except (KeyboardInterrupt, SystemExit, Exception) as exception: # ruff:ignore[blind-except] try: - if getattr(options, "with_traceback", False): - raise - if not (isinstance(exception, SystemExit) and exception.code == 0): - LOGGER.error("%s: %s", type(exception).__name__, exception) # noqa: TRY400 - code = exception.code if isinstance(exception, SystemExit) else 1 - sys.exit(code) + _exit_for_exception(options, exception) finally: for handler in LOGGER.handlers: # force flush of log messages before the trace is printed handler.flush() +def _exit_for_exception(options: VirtualEnvOptions, exception: BaseException) -> None: + if getattr(options, "with_traceback", False): + raise exception + if not (isinstance(exception, SystemExit) and exception.code == 0): + LOGGER.error("%s: %s", type(exception).__name__, exception) + sys.exit(exception.code if isinstance(exception, SystemExit) else 1) + + if __name__ == "__main__": # pragma: no cov run_with_catch() # pragma: no cov diff --git a/src/virtualenv/activation/__init__.py b/src/virtualenv/activation/__init__.py index 5aa4a9d76..2a1676d95 100644 --- a/src/virtualenv/activation/__init__.py +++ b/src/virtualenv/activation/__init__.py @@ -7,6 +7,7 @@ from .nushell import NushellActivator from .powershell import PowerShellActivator from .python import PythonActivator +from .xonsh import XonshActivator __all__ = [ "BashActivator", @@ -16,4 +17,5 @@ "NushellActivator", "PowerShellActivator", "PythonActivator", + "XonshActivator", ] diff --git a/src/virtualenv/activation/activator.py b/src/virtualenv/activation/activator.py index dd404b47c..251308ff4 100644 --- a/src/virtualenv/activation/activator.py +++ b/src/virtualenv/activation/activator.py @@ -2,45 +2,56 @@ import os from abc import ABC, abstractmethod +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from argparse import ArgumentParser + from pathlib import Path + + from python_discovery import PythonInfo + + from virtualenv.config.cli.parser import VirtualEnvOptions + from virtualenv.create.creator import Creator class Activator(ABC): """Generates activate script for the virtual environment.""" - def __init__(self, options) -> None: - """ - Create a new activator generator. + def __init__(self, options: VirtualEnvOptions) -> None: + """Create a new activator generator. :param options: the parsed options as defined within :meth:`add_parser_arguments` + """ self.flag_prompt = os.path.basename(os.getcwd()) if options.prompt == "." else options.prompt @classmethod - def supports(cls, interpreter): # noqa: ARG003 - """ - Check if the activation script is supported in the given interpreter. + def supports(cls, interpreter: PythonInfo) -> bool: # ruff:ignore[unused-class-method-argument] + """Check if the activation script is supported in the given interpreter. :param interpreter: the interpreter we need to support - :return: ``True`` if supported, ``False`` otherwise + + :returns: ``True`` if supported, ``False`` otherwise + """ return True - @classmethod # noqa: B027 - def add_parser_arguments(cls, parser, interpreter): - """ - Add CLI arguments for this activation script. + @classmethod # ruff:ignore[empty-method-without-abstract-decorator] + def add_parser_arguments(cls, parser: ArgumentParser, interpreter: PythonInfo) -> None: + """Add CLI arguments for this activation script. :param parser: the CLI parser :param interpreter: the interpreter this virtual environment is based of + """ @abstractmethod - def generate(self, creator): - """ - Generate activate script for the given creator. + def generate(self, creator: Creator) -> list[Path]: + """Generate activate script for the given creator. + + :param creator: the creator (based of :class:`virtualenv.create.creator.Creator`) we used to create this virtual + environment - :param creator: the creator (based of :class:`virtualenv.create.creator.Creator`) we used to create this \ - virtual environment """ raise NotImplementedError diff --git a/src/virtualenv/activation/bash/__init__.py b/src/virtualenv/activation/bash/__init__.py index 4f160744f..e083e9585 100644 --- a/src/virtualenv/activation/bash/__init__.py +++ b/src/virtualenv/activation/bash/__init__.py @@ -1,19 +1,25 @@ from __future__ import annotations from pathlib import Path +from typing import TYPE_CHECKING from virtualenv.activation.via_template import ViaTemplateActivator +if TYPE_CHECKING: + from collections.abc import Iterator + + from virtualenv.create.creator import Creator + class BashActivator(ViaTemplateActivator): - def templates(self): + def templates(self) -> Iterator[str]: yield "activate.sh" - def as_name(self, template): + def as_name(self, template: str) -> str: return Path(template).stem - def replacements(self, creator, dest): - data = super().replacements(creator, dest) + def replacements(self, creator: Creator, dest_folder: Path) -> dict[str, str]: + data = super().replacements(creator, dest_folder) data.update({ "__TCL_LIBRARY__": getattr(creator.interpreter, "tcl_lib", None) or "", "__TK_LIBRARY__": getattr(creator.interpreter, "tk_lib", None) or "", diff --git a/src/virtualenv/activation/bash/activate.sh b/src/virtualenv/activation/bash/activate.sh index de2d2422c..445bb3ce9 100644 --- a/src/virtualenv/activation/bash/activate.sh +++ b/src/virtualenv/activation/bash/activate.sh @@ -11,37 +11,56 @@ deactivate () { unset -f pydoc >/dev/null 2>&1 || true # reset old environment variables - # ! [ -z ${VAR+_} ] returns true if VAR is declared at all - if ! [ -z "${_OLD_VIRTUAL_PATH:+_}" ] ; then + if [ -n "${_OLD_VIRTUAL_PATH+x}" ] ; then PATH="$_OLD_VIRTUAL_PATH" export PATH unset _OLD_VIRTUAL_PATH fi - if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then + if [ -n "${_OLD_VIRTUAL_PYTHONHOME:-}" ] ; then PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME" export PYTHONHOME unset _OLD_VIRTUAL_PYTHONHOME fi - if ! [ -z "${_OLD_VIRTUAL_TCL_LIBRARY+_}" ]; then - TCL_LIBRARY="$_OLD_VIRTUAL_TCL_LIBRARY" - export TCL_LIBRARY + if [ -n "${_OLD_VIRTUAL_TCL_LIBRARY+x}" ]; then + if [ -n "$_OLD_VIRTUAL_TCL_LIBRARY" ]; then + TCL_LIBRARY="$_OLD_VIRTUAL_TCL_LIBRARY" + export TCL_LIBRARY + else + unset TCL_LIBRARY + fi unset _OLD_VIRTUAL_TCL_LIBRARY fi - if ! [ -z "${_OLD_VIRTUAL_TK_LIBRARY+_}" ]; then - TK_LIBRARY="$_OLD_VIRTUAL_TK_LIBRARY" - export TK_LIBRARY + if [ -n "${_OLD_VIRTUAL_TK_LIBRARY+x}" ]; then + if [ -n "$_OLD_VIRTUAL_TK_LIBRARY" ]; then + TK_LIBRARY="$_OLD_VIRTUAL_TK_LIBRARY" + export TK_LIBRARY + else + unset TK_LIBRARY + fi unset _OLD_VIRTUAL_TK_LIBRARY fi + if [ -n "${_OLD_PKG_CONFIG_PATH+x}" ]; then + if [ -n "$_OLD_PKG_CONFIG_PATH" ]; then + PKG_CONFIG_PATH="$_OLD_PKG_CONFIG_PATH" + export PKG_CONFIG_PATH + else + unset PKG_CONFIG_PATH + fi + unset _OLD_PKG_CONFIG_PATH + fi # The hash command must be called to get it to forget past # commands. Without forgetting past commands the $PATH changes # we made may not be respected - hash -r 2>/dev/null - - if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then - PS1="$_OLD_VIRTUAL_PS1" - export PS1 + hash -r 2>/dev/null || true + + if [ -n "${_OLD_VIRTUAL_PS1+x}" ] ; then + if [ -n "$_OLD_VIRTUAL_PS1" ] ; then + PS1="$_OLD_VIRTUAL_PS1" + else + unset PS1 + fi unset _OLD_VIRTUAL_PS1 fi @@ -57,23 +76,29 @@ deactivate () { deactivate nondestructive if [ ! -d __VIRTUAL_ENV__ ]; then - echo "Virtual environment directory __VIRTUAL_ENV__ does not exist!" >&2 - CURRENT_PATH=$(realpath "${0}") + echo "Virtual environment directory" __VIRTUAL_ENV__ "does not exist!" >&2 + CURRENT_PATH=$(realpath "${BASH_SOURCE[0]}") CURRENT_DIR=$(dirname "${CURRENT_PATH}") VIRTUAL_ENV="$(realpath "${CURRENT_DIR}/../")" else VIRTUAL_ENV=__VIRTUAL_ENV__ fi -if ([ "$OSTYPE" = "cygwin" ] || [ "$OSTYPE" = "msys" ]) && $(command -v cygpath &> /dev/null) ; then - VIRTUAL_ENV=$(cygpath -u "$VIRTUAL_ENV") -fi +case "$(uname)" in + CYGWIN*|MSYS*|MINGW*) + VIRTUAL_ENV=$(cygpath "$VIRTUAL_ENV") + ;; +esac export VIRTUAL_ENV _OLD_VIRTUAL_PATH="$PATH" PATH="$VIRTUAL_ENV/"__BIN_NAME__":$PATH" export PATH +_OLD_PKG_CONFIG_PATH="${PKG_CONFIG_PATH:-}" +PKG_CONFIG_PATH="${VIRTUAL_ENV}/lib/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}}" +export PKG_CONFIG_PATH + if [ "x"__VIRTUAL_PROMPT__ != x ] ; then VIRTUAL_ENV_PROMPT=__VIRTUAL_PROMPT__ else @@ -82,31 +107,47 @@ fi export VIRTUAL_ENV_PROMPT # unset PYTHONHOME if set -if ! [ -z "${PYTHONHOME+_}" ] ; then +if [ -n "${PYTHONHOME:-}" ] ; then _OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME" unset PYTHONHOME fi +# an empty saved value tells deactivate to unset the variable if [ __TCL_LIBRARY__ != "" ]; then - if ! [ -z "${TCL_LIBRARY+_}" ] ; then - _OLD_VIRTUAL_TCL_LIBRARY="$TCL_LIBRARY" - fi + _OLD_VIRTUAL_TCL_LIBRARY="${TCL_LIBRARY:-}" TCL_LIBRARY=__TCL_LIBRARY__ export TCL_LIBRARY fi if [ __TK_LIBRARY__ != "" ]; then - if ! [ -z "${TK_LIBRARY+_}" ] ; then - _OLD_VIRTUAL_TK_LIBRARY="$TK_LIBRARY" - fi + _OLD_VIRTUAL_TK_LIBRARY="${TK_LIBRARY:-}" TK_LIBRARY=__TK_LIBRARY__ export TK_LIBRARY fi if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then _OLD_VIRTUAL_PS1="${PS1-}" - PS1="(${VIRTUAL_ENV_PROMPT}) ${PS1-}" - export PS1 + _VIRTUAL_PROMPT="${VIRTUAL_ENV_PROMPT}" + # bash, and zsh under PROMPT_SUBST, re-expand backslashes, dollar signs and backticks in PS1 on every redraw; + # zsh gets the escaping even without the option, since a user can set it after activation. The guards keep + # the substitution away from POSIX shells such as dash, which reject it with "Bad substitution" + if [ -n "${BASH_VERSION-}" ]; then + # bash first decodes its backslash prompt escapes, turning \\ into \ and \$ into $ (or # for root), and only + # then expands the result, so each character needs escaping for both passes + _VIRTUAL_PROMPT="${_VIRTUAL_PROMPT//\\/\\\\\\\\}" + _VIRTUAL_PROMPT="${_VIRTUAL_PROMPT//\$/\\\\\$}" + _VIRTUAL_PROMPT="${_VIRTUAL_PROMPT//\`/\\\\\`}" + elif [ -n "${ZSH_VERSION-}" ]; then + _VIRTUAL_PROMPT="${_VIRTUAL_PROMPT//\\/\\\\}" + _VIRTUAL_PROMPT="${_VIRTUAL_PROMPT//\$/\\\$}" + _VIRTUAL_PROMPT="${_VIRTUAL_PROMPT//\`/\\\`}" + fi + # zsh expands % sequences in PS1 as prompt escapes + if [ -n "${ZSH_VERSION-}" ]; then + _VIRTUAL_PROMPT="${_VIRTUAL_PROMPT//\%/%%}" + fi + PS1="(${_VIRTUAL_PROMPT}) ${PS1-}" + unset _VIRTUAL_PROMPT fi # Make sure to unalias pydoc if it's already there diff --git a/src/virtualenv/activation/batch/__init__.py b/src/virtualenv/activation/batch/__init__.py index 3d74ba835..dc20a510a 100644 --- a/src/virtualenv/activation/batch/__init__.py +++ b/src/virtualenv/activation/batch/__init__.py @@ -1,30 +1,94 @@ from __future__ import annotations +import logging import os +from typing import TYPE_CHECKING, Final from virtualenv.activation.via_template import ViaTemplateActivator +from virtualenv.util.text import collapse_line_boundaries + +if TYPE_CHECKING: + from collections.abc import Iterator + from pathlib import Path + + from python_discovery import PythonInfo + + from virtualenv.create.creator import Creator + +LOGGER = logging.getLogger(__name__) + +# cmd.exe's own command-line parser looks for these regardless of surrounding quotes: confirmed on a +# real Windows runner that `@set "VAR=x & cmd"` still runs `cmd` as a second statement even though the +# whole VAR=value expression is quoted. `(` and `)` are not in this set - confirmed on the same runner +# that `@set "VAR=C:\Program Files (x86)\..."` round-trips unchanged, since parentheses are only special +# to cmd.exe as block delimiters in control-flow syntax (if/for), not as bare characters in a value. `!` +# is only live when the caller already has delayed expansion enabled (`setlocal enabledelayedexpansion`, +# `cmd /V:ON`) - not cmd.exe's default, but confirmed on a real runner that a prompt like `foo!SECRET!` +# then substitutes the actual value of an existing SECRET variable into the prompt, so it is neutered +# unconditionally rather than only when that mode happens to be on. +_CMD_OPERATORS: Final[tuple[str, ...]] = ("&", "|", "<", ">", "^", "!", '"') + +# These are valid filename characters that quote() would replace, changing the target directory. +_PATH_UNSAFE_CHARS: Final[tuple[str, ...]] = ("&", "^", "!") + +_PATH_REPLACEMENT_NAMES: Final[dict[str, str]] = { + "__VIRTUAL_ENV__": "the destination directory", + "__TCL_LIBRARY__": "the interpreter's Tcl library path", + "__TK_LIBRARY__": "the interpreter's Tk library path", +} class BatchActivator(ViaTemplateActivator): @classmethod - def supports(cls, interpreter): + def supports(cls, interpreter: PythonInfo) -> bool: return interpreter.os == "nt" - def templates(self): + def templates(self) -> Iterator[str]: yield "activate.bat" yield "deactivate.bat" yield "pydoc.bat" + def generate(self, creator: Creator) -> list[Path]: + values = super().replacements(creator, creator.bin_dir) + problems = [reason for key in _PATH_REPLACEMENT_NAMES if (reason := _unsafe_path_reason(key, values[key]))] + if problems: + for problem in problems: + LOGGER.warning("skipping batch activation scripts: %s", problem) + return [] + return super().generate(creator) + @staticmethod - def quote(string): - return string + def quote(string: str) -> str: + """Make a value safe to sit inside ``@set "VAR=value"``. + + Batch has no escape for a double quote in this context either: it always closes the quoted string, and whatever + follows on the line runs as live cmd.exe syntax. A raw line boundary is worse - batch is line-oriented + regardless of quote state, so it starts a brand-new statement instead of staying inside the value. None of these + can be represented literally here, so replace them with a space. ``%`` still triggers variable expansion inside + the quotes, but doubling it to ``%%`` is a real, in-file escape that keeps the literal character. - def instantiate_template(self, replacements, template, creator): + """ + string = string.replace("%", "%%") + for operator in _CMD_OPERATORS: + string = string.replace(operator, " ") + return collapse_line_boundaries(string) + + def instantiate_template(self, replacements: dict[str, str], template: str, creator: Creator) -> str: # ensure the text has all newlines as \r\n - required by batch base = super().instantiate_template(replacements, template, creator) return base.replace(os.linesep, "\n").replace("\n", os.linesep) +def _unsafe_path_reason(key: str, value: str) -> str | None: + found = sorted({char for char in value if char in _PATH_UNSAFE_CHARS}) + if not found: + return None + return ( + f"{_PATH_REPLACEMENT_NAMES[key]} ({value!r}) contains {''.join(found)!r}, which the batch activator " + "cannot preserve in generated scripts" + ) + + __all__ = [ "BatchActivator", ] diff --git a/src/virtualenv/activation/batch/activate.bat b/src/virtualenv/activation/batch/activate.bat index 62f393c80..7b9ac4302 100644 --- a/src/virtualenv/activation/batch/activate.bat +++ b/src/virtualenv/activation/batch/activate.bat @@ -5,6 +5,9 @@ "%SystemRoot%\System32\chcp.com" 65001 > nul ) +@REM Undo an activation that is still in effect, otherwise its values get saved as the ones to restore +@if defined _OLD_VIRTUAL_PATH @call "__VIRTUAL_ENV__\__BIN_NAME__\deactivate.bat" + @set "VIRTUAL_ENV=__VIRTUAL_ENV__" @set "VIRTUAL_ENV_PROMPT=__VIRTUAL_PROMPT__" @@ -39,6 +42,10 @@ @if defined TK_LIBRARY @set "_OLD_VIRTUAL_TK_LIBRARY=%TK_LIBRARY%" @if NOT "__TK_LIBRARY__"=="" @set "TK_LIBRARY=__TK_LIBRARY__" +@if defined PKG_CONFIG_PATH @set "_OLD_PKG_CONFIG_PATH=%PKG_CONFIG_PATH%" +@if defined PKG_CONFIG_PATH @set "PKG_CONFIG_PATH=%VIRTUAL_ENV%\lib\pkgconfig;%PKG_CONFIG_PATH%" +@if not defined PKG_CONFIG_PATH @set "PKG_CONFIG_PATH=%VIRTUAL_ENV%\lib\pkgconfig" + @REM if defined _OLD_VIRTUAL_PATH ( @if not defined _OLD_VIRTUAL_PATH @goto ENDIFVPATH1 @set "PATH=%_OLD_VIRTUAL_PATH%" diff --git a/src/virtualenv/activation/batch/deactivate.bat b/src/virtualenv/activation/batch/deactivate.bat index 7a12d47ed..d486958ad 100644 --- a/src/virtualenv/activation/batch/deactivate.bat +++ b/src/virtualenv/activation/batch/deactivate.bat @@ -20,6 +20,10 @@ @if not defined _OLD_VIRTUAL_TK_LIBRARY @set TK_LIBRARY= @set _OLD_VIRTUAL_TK_LIBRARY= +@if defined _OLD_PKG_CONFIG_PATH @set "PKG_CONFIG_PATH=%_OLD_PKG_CONFIG_PATH%" +@if not defined _OLD_PKG_CONFIG_PATH @set PKG_CONFIG_PATH= +@set _OLD_PKG_CONFIG_PATH= + @if not defined _OLD_VIRTUAL_PATH @goto ENDIFVPATH @set "PATH=%_OLD_VIRTUAL_PATH%" @set _OLD_VIRTUAL_PATH= diff --git a/src/virtualenv/activation/batch/pydoc.bat b/src/virtualenv/activation/batch/pydoc.bat index 3d46a231a..d55f8ac03 100644 --- a/src/virtualenv/activation/batch/pydoc.bat +++ b/src/virtualenv/activation/batch/pydoc.bat @@ -1 +1 @@ -python.exe -m pydoc %* +"python.exe" -m pydoc %* diff --git a/src/virtualenv/activation/cshell/__init__.py b/src/virtualenv/activation/cshell/__init__.py index 7001f999a..e8d38a7ba 100644 --- a/src/virtualenv/activation/cshell/__init__.py +++ b/src/virtualenv/activation/cshell/__init__.py @@ -1,15 +1,72 @@ from __future__ import annotations +import shlex +from typing import TYPE_CHECKING, Final + from virtualenv.activation.via_template import ViaTemplateActivator +if TYPE_CHECKING: + from collections.abc import Iterator + + from python_discovery import PythonInfo + + from virtualenv.create.creator import Creator + +_PROMPT_DISPLAY_TCSH: Final[str] = "__VIRTUAL_PROMPT_DISPLAY_TCSH__" +_PROMPT_DISPLAY_PLAIN: Final[str] = "__VIRTUAL_PROMPT_DISPLAY_PLAIN__" + class CShellActivator(ViaTemplateActivator): @classmethod - def supports(cls, interpreter): + def supports(cls, interpreter: PythonInfo) -> bool: return interpreter.os != "nt" - def templates(self): + def templates(self) -> Iterator[str]: yield "activate.csh" + yield "deactivate.csh" + + @staticmethod + def quote(string: str) -> str: + # csh runs history substitution before it parses quotes, so a ! survives shlex.quote's single + # quotes and aborts the whole script with "Event not found"; only a backslash suppresses it + return shlex.quote(string).replace("!", "\\!") + + def substitutions(self, replacements: dict[str, str], creator: Creator) -> dict[str, str]: + # carries its own quoting, so it cannot go through the shared quote() pass with the other replacements; + # tcsh and plain csh disagree on whether % needs escaping, so activate.csh picks one of these at runtime + display = creator.env_name if self.flag_prompt is None else self.flag_prompt + return { + **super().substitutions(replacements, creator), + _PROMPT_DISPLAY_TCSH: _prompt_literal(display, tcsh=True), + _PROMPT_DISPLAY_PLAIN: _prompt_literal(display, tcsh=False), + } + + +def _prompt_literal(value: str, *, tcsh: bool) -> str: + r"""Build csh source whose ``prompt`` renders ``value`` unchanged. + + Both csh and tcsh expand the prompt every time it draws it, and a literal ``!`` turns into the history event number + in either; only ``\!`` survives that pass, and it has to reach the variable as a real backslash, which costs a + second one in the source. ``%`` is different: tcsh treats a bare ``%`` as the start of an escape such as ``%p`` for + the time, and only a doubled ``%%`` survives as a literal percent, but plain csh has no such escape at all - a bare + ``%`` is already literal there, and doubling it would show two. Confirmed on tcsh 6.24 and on Debian's bsd-csh + package, which is what "csh" actually resolves to on Debian and Ubuntu. + + """ + parts = ["'"] + for char in value: + if char == "'": + parts.append("'\\''") + elif char == "!": + parts.append("\\\\!") + elif char == "%" and tcsh: + parts.append("%%") + elif char == "\\": + parts.append("\\\\") + else: + parts.append(char) + parts.append("'") + return "".join(parts) __all__ = [ diff --git a/src/virtualenv/activation/cshell/activate.csh b/src/virtualenv/activation/cshell/activate.csh index 5c02616d7..e6966696d 100644 --- a/src/virtualenv/activation/cshell/activate.csh +++ b/src/virtualenv/activation/cshell/activate.csh @@ -5,19 +5,27 @@ set newline='\ ' -alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PATH:q" && unset _OLD_VIRTUAL_PATH; rehash; test $?_OLD_VIRTUAL_TCL_LIBRARY != 0 && setenv TCL_LIBRARY "$_OLD_VIRTUAL_TCL_LIBRARY:q" && unset _OLD_VIRTUAL_TCL_LIBRARY || unsetenv TCL_LIBRARY; test $?_OLD_VIRTUAL_TK_LIBRARY != 0 && setenv TK_LIBRARY "$_OLD_VIRTUAL_TK_LIBRARY:q" && unset _OLD_VIRTUAL_TK_LIBRARY || unsetenv TK_LIBRARY; test $?_OLD_VIRTUAL_PROMPT != 0 && set prompt="$_OLD_VIRTUAL_PROMPT:q" && unset _OLD_VIRTUAL_PROMPT; unsetenv VIRTUAL_ENV; unsetenv VIRTUAL_ENV_PROMPT; test "\!:*" != "nondestructive" && unalias deactivate && unalias pydoc' - -# Unset irrelevant variables. -deactivate nondestructive +# Undo an activation that is still in effect, otherwise its values get saved as the ones to restore +if ($?_OLD_VIRTUAL_PATH) source __VIRTUAL_ENV__"/"__BIN_NAME__"/deactivate.csh" setenv VIRTUAL_ENV __VIRTUAL_ENV__ set _OLD_VIRTUAL_PATH="$PATH:q" setenv PATH "$VIRTUAL_ENV:q/"__BIN_NAME__":$PATH:q" +if ($?PKG_CONFIG_PATH) then + set _OLD_PKG_CONFIG_PATH="$PKG_CONFIG_PATH" + setenv PKG_CONFIG_PATH "${VIRTUAL_ENV}/lib/pkgconfig:${PKG_CONFIG_PATH}" +else + set _OLD_PKG_CONFIG_PATH="" + setenv PKG_CONFIG_PATH "${VIRTUAL_ENV}/lib/pkgconfig" +endif + if (__TCL_LIBRARY__ != "") then if ($?TCL_LIBRARY) then set _OLD_VIRTUAL_TCL_LIBRARY="$TCL_LIBRARY" + else + set _OLD_VIRTUAL_TCL_LIBRARY="" endif setenv TCL_LIBRARY __TCL_LIBRARY__ endif @@ -25,6 +33,8 @@ endif if (__TK_LIBRARY__ != "") then if ($?TK_LIBRARY) then set _OLD_VIRTUAL_TK_LIBRARY="$TK_LIBRARY" + else + set _OLD_VIRTUAL_TK_LIBRARY="" endif setenv TK_LIBRARY __TK_LIBRARY__ endif @@ -53,8 +63,10 @@ if ( $do_prompt == "1" ) then set _OLD_VIRTUAL_PROMPT="$prompt:q" if ( "$prompt:q" =~ *"$newline:q"* ) then : + else if ( $?tcsh ) then + set prompt = '('__VIRTUAL_PROMPT_DISPLAY_TCSH__') '"$prompt:q" else - set prompt = '('"$VIRTUAL_ENV_PROMPT:q"') '"$prompt:q" + set prompt = '('__VIRTUAL_PROMPT_DISPLAY_PLAIN__') '"$prompt:q" endif endif endif @@ -64,4 +76,9 @@ unset do_prompt alias pydoc python -m pydoc +# stashed in a variable, not embedded directly in the alias body, because the alias's own single +# quotes cannot nest around a path that itself needed single-quoting +set deactivate_script = __VIRTUAL_ENV__"/"__BIN_NAME__"/deactivate.csh" +alias deactivate 'source "$deactivate_script:q"' + rehash diff --git a/src/virtualenv/activation/cshell/deactivate.csh b/src/virtualenv/activation/cshell/deactivate.csh new file mode 100644 index 000000000..e56172750 --- /dev/null +++ b/src/virtualenv/activation/cshell/deactivate.csh @@ -0,0 +1,46 @@ +# Sourced by the deactivate alias in activate.csh. Do not run it directly. + +if ($?_OLD_VIRTUAL_PATH) then + setenv PATH "$_OLD_VIRTUAL_PATH:q" + unset _OLD_VIRTUAL_PATH +endif +rehash + +# an empty saved value tells us the variable was unset before activation +if ($?_OLD_VIRTUAL_TCL_LIBRARY) then + if ("$_OLD_VIRTUAL_TCL_LIBRARY:q" == "") then + unsetenv TCL_LIBRARY + else + setenv TCL_LIBRARY "$_OLD_VIRTUAL_TCL_LIBRARY:q" + endif + unset _OLD_VIRTUAL_TCL_LIBRARY +endif + +if ($?_OLD_VIRTUAL_TK_LIBRARY) then + if ("$_OLD_VIRTUAL_TK_LIBRARY:q" == "") then + unsetenv TK_LIBRARY + else + setenv TK_LIBRARY "$_OLD_VIRTUAL_TK_LIBRARY:q" + endif + unset _OLD_VIRTUAL_TK_LIBRARY +endif + +if ($?_OLD_PKG_CONFIG_PATH) then + if ("$_OLD_PKG_CONFIG_PATH:q" == "") then + unsetenv PKG_CONFIG_PATH + else + setenv PKG_CONFIG_PATH "$_OLD_PKG_CONFIG_PATH:q" + endif + unset _OLD_PKG_CONFIG_PATH +endif + +if ($?_OLD_VIRTUAL_PROMPT) then + set prompt="$_OLD_VIRTUAL_PROMPT:q" + unset _OLD_VIRTUAL_PROMPT +endif + +unsetenv VIRTUAL_ENV +unsetenv VIRTUAL_ENV_PROMPT +unalias deactivate +unalias pydoc +unset deactivate_script diff --git a/src/virtualenv/activation/fish/__init__.py b/src/virtualenv/activation/fish/__init__.py index 26263566e..81d6d9219 100644 --- a/src/virtualenv/activation/fish/__init__.py +++ b/src/virtualenv/activation/fish/__init__.py @@ -1,14 +1,28 @@ from __future__ import annotations +from typing import TYPE_CHECKING + from virtualenv.activation.via_template import ViaTemplateActivator +if TYPE_CHECKING: + from collections.abc import Iterator + from pathlib import Path + + from virtualenv.create.creator import Creator + class FishActivator(ViaTemplateActivator): - def templates(self): + def templates(self) -> Iterator[str]: yield "activate.fish" - def replacements(self, creator, dest): - data = super().replacements(creator, dest) + @staticmethod + def quote(string: str) -> str: + # fish reads \\ and \' as escapes inside single quotes, unlike POSIX sh, so shlex.quote's output lets a + # backslash before a quote end the literal early + return "'" + string.replace("\\", "\\\\").replace("'", "\\'") + "'" + + def replacements(self, creator: Creator, dest_folder: Path) -> dict[str, str]: + data = super().replacements(creator, dest_folder) data.update({ "__TCL_LIBRARY__": getattr(creator.interpreter, "tcl_lib", None) or "", "__TK_LIBRARY__": getattr(creator.interpreter, "tk_lib", None) or "", diff --git a/src/virtualenv/activation/fish/activate.fish b/src/virtualenv/activation/fish/activate.fish index c9d174997..3bec218a7 100644 --- a/src/virtualenv/activation/fish/activate.fish +++ b/src/virtualenv/activation/fish/activate.fish @@ -1,46 +1,37 @@ # This file must be used using `source bin/activate.fish` *within a running fish ( http://fishshell.com ) session*. # Do not run it directly. -function _bashify_path -d "Converts a fish path to something bash can recognize" - set fishy_path $argv - set bashy_path $fishy_path[1] - for path_part in $fishy_path[2..-1] - set bashy_path "$bashy_path:$path_part" - end - echo $bashy_path -end - -function _fishify_path -d "Converts a bash path to something fish can recognize" - echo $argv | tr ':' '\n' -end - function deactivate -d 'Exit virtualenv mode and return to the normal environment.' # reset old environment variables - if test -n "$_OLD_VIRTUAL_PATH" - # https://github.com/fish-shell/fish-shell/issues/436 altered PATH handling - if test (string sub -s 1 -l 1 $FISH_VERSION) -lt 3 - set -gx PATH (_fishify_path "$_OLD_VIRTUAL_PATH") - else - set -gx PATH $_OLD_VIRTUAL_PATH - end + if set -q _OLD_VIRTUAL_PATH + set -gx PATH $_OLD_VIRTUAL_PATH set -e _OLD_VIRTUAL_PATH end - if test -n __TCL_LIBRARY__ - if test -n "$_OLD_VIRTUAL_TCL_LIBRARY"; - set -gx TCL_LIBRARY "$_OLD_VIRTUAL_TCL_LIBRARY"; - set -e _OLD_VIRTUAL_TCL_LIBRARY; - else; - set -e TCL_LIBRARY; - end + if set -q _OLD_VIRTUAL_TCL_LIBRARY + if test -n "$_OLD_VIRTUAL_TCL_LIBRARY" + set -gx TCL_LIBRARY "$_OLD_VIRTUAL_TCL_LIBRARY" + else + set -e TCL_LIBRARY + end + set -e _OLD_VIRTUAL_TCL_LIBRARY end - if test -n __TK_LIBRARY__ - if test -n "$_OLD_VIRTUAL_TK_LIBRARY"; - set -gx TK_LIBRARY "$_OLD_VIRTUAL_TK_LIBRARY"; - set -e _OLD_VIRTUAL_TK_LIBRARY; - else; - set -e TK_LIBRARY; - end + if set -q _OLD_VIRTUAL_TK_LIBRARY + if test -n "$_OLD_VIRTUAL_TK_LIBRARY" + set -gx TK_LIBRARY "$_OLD_VIRTUAL_TK_LIBRARY" + else + set -e TK_LIBRARY + end + set -e _OLD_VIRTUAL_TK_LIBRARY + end + + if set -q _OLD_PKG_CONFIG_PATH + if test -n "$_OLD_PKG_CONFIG_PATH" + set -gx PKG_CONFIG_PATH "$_OLD_PKG_CONFIG_PATH" + else + set -e PKG_CONFIG_PATH + end + set -e _OLD_PKG_CONFIG_PATH end if test -n "$_OLD_VIRTUAL_PYTHONHOME" @@ -49,14 +40,14 @@ function deactivate -d 'Exit virtualenv mode and return to the normal environmen end if test -n "$_OLD_FISH_PROMPT_OVERRIDE" - and functions -q _old_fish_prompt + and builtin functions -q _old_fish_prompt # Set an empty local `$fish_function_path` to allow the removal of `fish_prompt` using `functions -e`. set -l fish_function_path # Erase virtualenv's `fish_prompt` and restore the original. - functions -e fish_prompt - functions -c _old_fish_prompt fish_prompt - functions -e _old_fish_prompt + builtin functions -e fish_prompt + builtin functions -c _old_fish_prompt fish_prompt + builtin functions -e _old_fish_prompt set -e _OLD_FISH_PROMPT_OVERRIDE end @@ -65,10 +56,8 @@ function deactivate -d 'Exit virtualenv mode and return to the normal environmen if test "$argv[1]" != 'nondestructive' # Self-destruct! - functions -e pydoc - functions -e deactivate - functions -e _bashify_path - functions -e _fishify_path + builtin functions -e pydoc + builtin functions -e deactivate end end @@ -76,26 +65,28 @@ end deactivate nondestructive set -gx VIRTUAL_ENV __VIRTUAL_ENV__ +if string match -qr 'CYGWIN|MSYS|MINGW' (uname) + set -gx VIRTUAL_ENV (cygpath -u $VIRTUAL_ENV) +end -# https://github.com/fish-shell/fish-shell/issues/436 altered PATH handling -if test (string sub -s 1 -l 1 $FISH_VERSION) -lt 3 - set -gx _OLD_VIRTUAL_PATH (_bashify_path $PATH) +set -gx _OLD_PKG_CONFIG_PATH "$PKG_CONFIG_PATH" +if test -n "$PKG_CONFIG_PATH" + set -gx PKG_CONFIG_PATH "$VIRTUAL_ENV/lib/pkgconfig:$PKG_CONFIG_PATH" else - set -gx _OLD_VIRTUAL_PATH $PATH + set -gx PKG_CONFIG_PATH "$VIRTUAL_ENV/lib/pkgconfig" end + +set -gx _OLD_VIRTUAL_PATH $PATH set -gx PATH "$VIRTUAL_ENV"'/'__BIN_NAME__ $PATH +# an empty saved value tells deactivate to erase the variable if test -n __TCL_LIBRARY__ - if set -q TCL_LIBRARY; - set -gx _OLD_VIRTUAL_TCL_LIBRARY $TCL_LIBRARY; - end - set -gx TCL_LIBRARY '__TCL_LIBRARY__' + set -gx _OLD_VIRTUAL_TCL_LIBRARY "$TCL_LIBRARY" + set -gx TCL_LIBRARY __TCL_LIBRARY__ end if test -n __TK_LIBRARY__ - if set -q TK_LIBRARY; - set -gx _OLD_VIRTUAL_TK_LIBRARY $TK_LIBRARY; - end - set -gx TK_LIBRARY '__TK_LIBRARY__' + set -gx _OLD_VIRTUAL_TK_LIBRARY "$TK_LIBRARY" + set -gx TK_LIBRARY __TK_LIBRARY__ end # Prompt override provided? @@ -118,15 +109,20 @@ end if test -z "$VIRTUAL_ENV_DISABLE_PROMPT" # Copy the current `fish_prompt` function as `_old_fish_prompt`. - functions -c fish_prompt _old_fish_prompt + builtin functions -c fish_prompt _old_fish_prompt + # Route every builtin the prompt path uses through `builtin` so a user function that shadows + # `printf`, `string`, `echo`, or `source`/`.` cannot hijack the prompt (Issue #93858 follow-up, + # CPython gh-140006). A dot-style directory navigator redefining `.` is the common trigger. function fish_prompt + set -l old_status $status # Run the user's prompt first; it might depend on (pipe)status. set -l prompt (_old_fish_prompt) - printf '(%s) ' $VIRTUAL_ENV_PROMPT + builtin printf '(%s) ' $VIRTUAL_ENV_PROMPT - string join -- \n $prompt # handle multi-line prompts + builtin string join -- \n $prompt # handle multi-line prompts + builtin echo "exit $old_status" | builtin source - end set -gx _OLD_FISH_PROMPT_OVERRIDE "$VIRTUAL_ENV" diff --git a/src/virtualenv/activation/nushell/__init__.py b/src/virtualenv/activation/nushell/__init__.py index d3b312497..0d5dd54c3 100644 --- a/src/virtualenv/activation/nushell/__init__.py +++ b/src/virtualenv/activation/nushell/__init__.py @@ -1,21 +1,28 @@ from __future__ import annotations +from typing import TYPE_CHECKING + from virtualenv.activation.via_template import ViaTemplateActivator +if TYPE_CHECKING: + from collections.abc import Iterator + from pathlib import Path + + from virtualenv.create.creator import Creator + class NushellActivator(ViaTemplateActivator): - def templates(self): + def templates(self) -> Iterator[str]: yield "activate.nu" @staticmethod - def quote(string): - """ - Nushell supports raw strings like: r###'this is a string'###. + def quote(string: str) -> str: + """Nushell supports raw strings like: r###'this is a string'###. https://github.com/nushell/nushell.github.io/blob/main/book/working_with_strings.md - This method finds the maximum continuous sharps in the string and then - quote it with an extra sharp. + This method finds the maximum continuous sharps in the string and then quote it with an extra sharp. + """ max_sharps = 0 current_sharps = 0 @@ -28,7 +35,7 @@ def quote(string): wrapping = "#" * (max_sharps + 1) return f"r{wrapping}'{string}'{wrapping}" - def replacements(self, creator, dest_folder): # noqa: ARG002 + def replacements(self, creator: Creator, dest_folder: Path) -> dict[str, str]: # ruff:ignore[unused-method-argument] return { "__VIRTUAL_PROMPT__": "" if self.flag_prompt is None else self.flag_prompt, "__VIRTUAL_ENV__": str(creator.dest), diff --git a/src/virtualenv/activation/nushell/activate.nu b/src/virtualenv/activation/nushell/activate.nu index b48fdd03f..6761921d0 100644 --- a/src/virtualenv/activation/nushell/activate.nu +++ b/src/virtualenv/activation/nushell/activate.nu @@ -2,8 +2,9 @@ # - Activate with `overlay use activate.nu` # - Deactivate with `deactivate`, as usual # -# To customize the overlay name, you can call `overlay use activate.nu as foo`, but then simply `deactivate` won't work -# because it is just an alias to hide the "activate" overlay. You'd need to call `overlay hide foo` manually. +# To customize the overlay name, call `overlay use activate.nu as foo`, but then `deactivate` won't work — +# you'd need to run `overlay hide foo` manually. Do not activate with `use activate.nu *`; that does not +# create an overlay and deactivation will fail. module warning { export-env { @@ -57,7 +58,14 @@ export-env { } else { __VIRTUAL_PROMPT__ } - let new_env = { $path_name: $new_path VIRTUAL_ENV: $virtual_env VIRTUAL_ENV_PROMPT: $virtual_env_prompt } + let old_pkg_config_path = if (has-env 'PKG_CONFIG_PATH') { $env.PKG_CONFIG_PATH } else { '' } + let venv_pkg_config_path = ([$virtual_env lib pkgconfig] | path join) + let new_pkg_config_path = if ($old_pkg_config_path | is-empty) { + $venv_pkg_config_path + } else { + $'($venv_pkg_config_path)(char esep)($old_pkg_config_path)' + } + let new_env = { $path_name: $new_path VIRTUAL_ENV: $virtual_env VIRTUAL_ENV_PROMPT: $virtual_env_prompt PKG_CONFIG_PATH: $new_pkg_config_path } if (has-env 'TCL_LIBRARY') { let $new_env = $new_env | insert TCL_LIBRARY __TCL_LIBRARY__ } @@ -84,4 +92,5 @@ export-env { } export alias pydoc = python -m pydoc +# If deactivate errors "not an active overlay": activate with `overlay use activate.nu` not `use activate.nu *`; for custom names use `overlay hide NAME` export alias deactivate = overlay hide activate diff --git a/src/virtualenv/activation/powershell/__init__.py b/src/virtualenv/activation/powershell/__init__.py index 8489656cc..e25290571 100644 --- a/src/virtualenv/activation/powershell/__init__.py +++ b/src/virtualenv/activation/powershell/__init__.py @@ -1,22 +1,31 @@ from __future__ import annotations +from typing import TYPE_CHECKING + from virtualenv.activation.via_template import ViaTemplateActivator +if TYPE_CHECKING: + from collections.abc import Iterator + class PowerShellActivator(ViaTemplateActivator): - def templates(self): + def templates(self) -> Iterator[str]: yield "activate.ps1" @staticmethod - def quote(string): - """ - This should satisfy PowerShell quoting rules [1], unless the quoted - string is passed directly to Windows native commands [2]. + def quote(string: str) -> str: + """This should satisfy PowerShell quoting rules [1], unless the quoted string is passed directly to Windows native commands [2]. [1]: https://learn.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_quoting_rules - [2]: https://learn.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_parsing#passing-arguments-that-contain-quote-characters - """ # noqa: D205 - string = string.replace("'", "''") + [2]: + https://learn.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_parsing#passing-arguments-that-contain-quote-characters + + """ + # PowerShell treats the ASCII apostrophe and the curly single quotes U+2018-U+201B as + # interchangeable single-quote delimiters, so each must be doubled to stay inert inside the literal. + for point in (0x27, 0x2018, 0x2019, 0x201A, 0x201B): + delimiter = chr(point) + string = string.replace(delimiter, delimiter * 2) return f"'{string}'" diff --git a/src/virtualenv/activation/powershell/activate.ps1 b/src/virtualenv/activation/powershell/activate.ps1 index 9f95e4370..bfa473bce 100644 --- a/src/virtualenv/activation/powershell/activate.ps1 +++ b/src/virtualenv/activation/powershell/activate.ps1 @@ -1,5 +1,92 @@ -$script:THIS_PATH = $myinvocation.mycommand.path -$script:BASE_DIR = Split-Path (Resolve-Path "$THIS_PATH/..") -Parent +<# +.Synopsis +Activate a Python virtual environment for the current PowerShell session. + +.Description +Pushes the python executable for a virtual environment to the front of the +$Env:PATH environment variable and sets the prompt to signify that you are +in a Python virtual environment. Makes use of the command line switches as +well as the `pyvenv.cfg` file values present in the virtual environment. + +.Parameter VenvDir +Path to the directory that contains the virtual environment to activate. The +default value for this is the parent of the directory that the activate.ps1 +script is located within. + +.Parameter Prompt +The prompt prefix to display when this virtual environment is activated. By +default, this prompt is the name of the virtual environment folder (VenvDir) +surrounded by parentheses and followed by a single space (ie. '(.venv) '). + +.Example +activate.ps1 +Activates the Python virtual environment that contains the activate.ps1 script. + +.Example +activate.ps1 -Verbose +Activates the Python virtual environment that contains the activate.ps1 script, +and shows extra information about the activation as it executes. + +.Example +activate.ps1 -VenvDir C:\Users\MyUser\Common\.venv +Activates the Python virtual environment located in the specified location. + +.Example +activate.ps1 -Prompt "MyPython" +Activates the Python virtual environment that contains the activate.ps1 script, +and prefixes the current prompt with the specified string (surrounded in +parentheses) while the virtual environment is active. + +.Notes +On Windows, it may be required to enable this activate.ps1 script by setting the +execution policy for the user. You can do this by issuing the following PowerShell +command: + +PS C:\> Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser + +For more information on Execution Policies: +https://go.microsoft.com/fwlink/?LinkID=135170 + +#> +Param( + [Parameter(Mandatory = $false)] + [String] + $VenvDir, + [Parameter(Mandatory = $false)] + [String] + $Prompt +) + +function Get-PyVenvConfig( + [String] + $ConfigDir +) { + Write-Verbose "Given ConfigDir=$ConfigDir, obtain values in pyvenv.cfg" + + $pyvenvConfigPath = Join-Path -Resolve -Path $ConfigDir -ChildPath 'pyvenv.cfg' -ErrorAction Continue + + $pyvenvConfig = @{ } + + if ($pyvenvConfigPath) { + Write-Verbose "File exists, parse ``key = value`` lines" + $pyvenvConfigContent = Get-Content -Path $pyvenvConfigPath + + $pyvenvConfigContent | ForEach-Object { + $keyval = $PSItem -split "\s*=\s*", 2 + if ($keyval[0] -and $keyval[1]) { + $val = $keyval[1] + + if ("'""".Contains($val.Substring(0, 1))) { + $val = $val.Substring(1, $val.Length - 2) + } + + $pyvenvConfig[$keyval[0]] = $val + Write-Verbose "Adding Key: '$($keyval[0])'='$val'" + } + } + } + return $pyvenvConfig +} function global:deactivate([switch] $NonDestructive) { if (Test-Path variable:_OLD_VIRTUAL_PATH) { @@ -10,19 +97,16 @@ function global:deactivate([switch] $NonDestructive) { if (Test-Path variable:_OLD_VIRTUAL_TCL_LIBRARY) { $env:TCL_LIBRARY = $variable:_OLD_VIRTUAL_TCL_LIBRARY Remove-Variable "_OLD_VIRTUAL_TCL_LIBRARY" -Scope global - } else { - if (Test-Path env:TCL_LIBRARY) { - Remove-Item env:TCL_LIBRARY -ErrorAction SilentlyContinue - } } if (Test-Path variable:_OLD_VIRTUAL_TK_LIBRARY) { $env:TK_LIBRARY = $variable:_OLD_VIRTUAL_TK_LIBRARY Remove-Variable "_OLD_VIRTUAL_TK_LIBRARY" -Scope global - } else { - if (Test-Path env:TK_LIBRARY) { - Remove-Item env:TK_LIBRARY -ErrorAction SilentlyContinue - } + } + + if (Test-Path variable:_OLD_PKG_CONFIG_PATH) { + $env:PKG_CONFIG_PATH = $variable:_OLD_PKG_CONFIG_PATH + Remove-Variable "_OLD_PKG_CONFIG_PATH" -Scope global } if (Test-Path function:_old_virtual_prompt) { @@ -38,56 +122,82 @@ function global:deactivate([switch] $NonDestructive) { Remove-Item env:VIRTUAL_ENV_PROMPT -ErrorAction SilentlyContinue } + if (Get-Variable -Name "_PYTHON_VENV_PROMPT_PREFIX" -ErrorAction SilentlyContinue) { + Remove-Variable -Name _PYTHON_VENV_PROMPT_PREFIX -Scope Global -Force + } + if (!$NonDestructive) { - # Self destruct! Remove-Item function:deactivate Remove-Item function:pydoc } } function global:pydoc { - python -m pydoc $args + & python -m pydoc $args } -# unset irrelevant variables -deactivate -nondestructive +$VenvExecPath = Split-Path -Parent $MyInvocation.MyCommand.Definition +$VenvExecDir = Get-Item -Path $VenvExecPath -$VIRTUAL_ENV = $BASE_DIR -$env:VIRTUAL_ENV = $VIRTUAL_ENV +Write-Verbose "Activation script is located in path: '$VenvExecPath'" -if (__VIRTUAL_PROMPT__ -ne "") { - $env:VIRTUAL_ENV_PROMPT = __VIRTUAL_PROMPT__ +if ($VenvDir) { + Write-Verbose "VenvDir given as parameter, using '$VenvDir' to determine values" +} else { + $VenvDir = $VenvExecDir.Parent.FullName.TrimEnd("\\/") + Write-Verbose "VenvDir=$VenvDir" } -else { - $env:VIRTUAL_ENV_PROMPT = $( Split-Path $env:VIRTUAL_ENV -Leaf ) + +$pyvenvCfg = Get-PyVenvConfig -ConfigDir $VenvDir + +if ($Prompt) { + Write-Verbose "Prompt specified as argument, using '$Prompt'" +} else { + if ($pyvenvCfg -and $pyvenvCfg['prompt']) { + Write-Verbose "Setting based on value in pyvenv.cfg='$($pyvenvCfg['prompt'])'" + $Prompt = $pyvenvCfg['prompt'] + } elseif (__VIRTUAL_PROMPT__ -ne "") { + $Prompt = __VIRTUAL_PROMPT__ + } else { + $Prompt = Split-Path -Path $VenvDir -Leaf + } } +deactivate -nondestructive + +$env:VIRTUAL_ENV = $VenvDir +$env:VIRTUAL_ENV_PROMPT = $Prompt + +# a $null saved value tells deactivate to remove the variable if (__TCL_LIBRARY__ -ne "") { - if (Test-Path env:TCL_LIBRARY) { - New-Variable -Scope global -Name _OLD_VIRTUAL_TCL_LIBRARY -Value $env:TCL_LIBRARY - } + New-Variable -Scope global -Name _OLD_VIRTUAL_TCL_LIBRARY -Value $env:TCL_LIBRARY $env:TCL_LIBRARY = __TCL_LIBRARY__ } if (__TK_LIBRARY__ -ne "") { - if (Test-Path env:TK_LIBRARY) { - New-Variable -Scope global -Name _OLD_VIRTUAL_TK_LIBRARY -Value $env:TK_LIBRARY - } + New-Variable -Scope global -Name _OLD_VIRTUAL_TK_LIBRARY -Value $env:TK_LIBRARY $env:TK_LIBRARY = __TK_LIBRARY__ } New-Variable -Scope global -Name _OLD_VIRTUAL_PATH -Value $env:PATH +New-Variable -Scope global -Name _OLD_PKG_CONFIG_PATH -Value $env:PKG_CONFIG_PATH +$env:PKG_CONFIG_PATH = [IO.Path]::Combine($env:VIRTUAL_ENV, "lib", "pkgconfig") +if ($_OLD_PKG_CONFIG_PATH) { + $env:PKG_CONFIG_PATH += __PATH_SEP__ + $_OLD_PKG_CONFIG_PATH +} + $env:PATH = "$env:VIRTUAL_ENV/" + __BIN_NAME__ + __PATH_SEP__ + $env:PATH + if (!$env:VIRTUAL_ENV_DISABLE_PROMPT) { function global:_old_virtual_prompt { "" } $function:_old_virtual_prompt = $function:prompt + New-Variable -Name _PYTHON_VENV_PROMPT_PREFIX -Description "Python virtual environment prompt prefix" -Scope Global -Option ReadOnly -Visibility Public -Value $Prompt function global:prompt { - # Add the custom prefix to the existing prompt - $previous_prompt_value = & $function:_old_virtual_prompt - ("(" + $env:VIRTUAL_ENV_PROMPT + ") " + $previous_prompt_value) + Write-Host -NoNewline -ForegroundColor Green "($($_PYTHON_VENV_PROMPT_PREFIX)) " + _old_virtual_prompt } } diff --git a/src/virtualenv/activation/python/__init__.py b/src/virtualenv/activation/python/__init__.py index e900f7ec9..ef59448b4 100644 --- a/src/virtualenv/activation/python/__init__.py +++ b/src/virtualenv/activation/python/__init__.py @@ -2,19 +2,26 @@ import os from collections import OrderedDict +from typing import TYPE_CHECKING from virtualenv.activation.via_template import ViaTemplateActivator +if TYPE_CHECKING: + from collections.abc import Iterator + from pathlib import Path + + from virtualenv.create.creator import Creator + class PythonActivator(ViaTemplateActivator): - def templates(self): + def templates(self) -> Iterator[str]: yield "activate_this.py" @staticmethod - def quote(string): + def quote(string: str) -> str: return repr(string) - def replacements(self, creator, dest_folder): + def replacements(self, creator: Creator, dest_folder: Path) -> dict[str, str]: replacements = super().replacements(creator, dest_folder) lib_folders = OrderedDict((os.path.relpath(str(i), str(dest_folder)), None) for i in creator.libs) lib_folders = os.pathsep.join(lib_folders.keys()) diff --git a/src/virtualenv/activation/python/activate_this.py b/src/virtualenv/activation/python/activate_this.py index 9cc816fab..7ce6a6921 100644 --- a/src/virtualenv/activation/python/activate_this.py +++ b/src/virtualenv/activation/python/activate_this.py @@ -1,11 +1,10 @@ -""" -Activate virtualenv for current interpreter: +"""Activate virtualenv for current interpreter: -import runpy -runpy.run_path(this_file) +import runpy runpy.run_path(this_file) This can be used when you must use an existing Python interpreter, not the virtualenv bin/python. -""" # noqa: D415 + +""" # ruff:ignore[missing-terminal-punctuation] from __future__ import annotations @@ -20,19 +19,27 @@ raise AssertionError(msg) from exc bin_dir = os.path.dirname(abs_file) -base = bin_dir[: -len(__BIN_NAME__) - 1] # strip away the bin part from the __file__, plus the path separator +base = bin_dir[: -len(__BIN_NAME__) - 1] # ty: ignore[unresolved-reference] # prepend bin to PATH (this file is inside the bin directory) os.environ["PATH"] = os.pathsep.join([bin_dir, *os.environ.get("PATH", "").split(os.pathsep)]) os.environ["VIRTUAL_ENV"] = base # virtual env is right above bin directory -os.environ["VIRTUAL_ENV_PROMPT"] = __VIRTUAL_PROMPT__ or os.path.basename(base) +os.environ["VIRTUAL_ENV_PROMPT"] = __VIRTUAL_PROMPT__ or os.path.basename(base) # ty: ignore[unresolved-reference] + +# Set PKG_CONFIG_PATH to include the virtualenv's pkgconfig directory +pkg_config_path = os.path.join(base, "lib", "pkgconfig") +existing_pkg_config_path = os.environ.get("PKG_CONFIG_PATH", "") +if existing_pkg_config_path: + os.environ["PKG_CONFIG_PATH"] = os.pathsep.join([pkg_config_path, existing_pkg_config_path]) +else: + os.environ["PKG_CONFIG_PATH"] = pkg_config_path # add the virtual environments libraries to the host python import mechanism prev_length = len(sys.path) -for lib in __LIB_FOLDERS__.split(os.pathsep): +for lib in __LIB_FOLDERS__.split(os.pathsep): # ty: ignore[unresolved-reference] path = os.path.realpath(os.path.join(bin_dir, lib)) - site.addsitedir(path.decode("utf-8") if __DECODE_PATH__ else path) + site.addsitedir(path.decode("utf-8") if __DECODE_PATH__ else path) # ty: ignore[unresolved-reference,unresolved-attribute] sys.path[:] = sys.path[prev_length:] + sys.path[0:prev_length] -sys.real_prefix = sys.prefix +sys.real_prefix = sys.prefix # ty: ignore[unresolved-attribute] sys.prefix = base diff --git a/src/virtualenv/activation/via_template.py b/src/virtualenv/activation/via_template.py index 85f932605..66ca61f23 100644 --- a/src/virtualenv/activation/via_template.py +++ b/src/virtualenv/activation/via_template.py @@ -1,12 +1,20 @@ from __future__ import annotations import os +import re import shlex import sys from abc import ABC, abstractmethod +from typing import TYPE_CHECKING from .activator import Activator +if TYPE_CHECKING: + from collections.abc import Iterable, Iterator + from pathlib import Path + + from virtualenv.create.creator import Creator + if sys.version_info >= (3, 10): from importlib.resources import files @@ -19,20 +27,21 @@ def read_binary(module_name: str, filename: str) -> bytes: class ViaTemplateActivator(Activator, ABC): @abstractmethod - def templates(self): + def templates(self) -> Iterator[str]: raise NotImplementedError @staticmethod - def quote(string): - """ - Quote strings in the activation script. + def quote(string: str) -> str: + """Quote strings in the activation script. :param string: the string to quote - :return: quoted string that works in the activation script + + :returns: quoted string that works in the activation script + """ return shlex.quote(string) - def generate(self, creator): + def generate(self, creator: Creator) -> list[Path]: dest_folder = creator.bin_dir replacements = self.replacements(creator, dest_folder) generated = self._generate(replacements, self.templates(), dest_folder, creator) @@ -40,7 +49,7 @@ def generate(self, creator): creator.pyenv_cfg["prompt"] = self.flag_prompt return generated - def replacements(self, creator, dest_folder): # noqa: ARG002 + def replacements(self, creator: Creator, dest_folder: Path) -> dict[str, str]: # ruff:ignore[unused-method-argument] return { "__VIRTUAL_PROMPT__": "" if self.flag_prompt is None else self.flag_prompt, "__VIRTUAL_ENV__": str(creator.dest), @@ -51,7 +60,9 @@ def replacements(self, creator, dest_folder): # noqa: ARG002 "__TK_LIBRARY__": getattr(creator.interpreter, "tk_lib", None) or "", } - def _generate(self, replacements, templates, to_folder, creator): + def _generate( + self, replacements: dict[str, str], templates: Iterable[str], to_folder: Path, creator: Creator + ) -> list[Path]: generated = [] for template in templates: text = self.instantiate_template(replacements, template, creator) @@ -67,20 +78,32 @@ def _generate(self, replacements, templates, to_folder, creator): generated.append(dest) return generated - def as_name(self, template): + def as_name(self, template: str) -> str: return template - def instantiate_template(self, replacements, template, creator): + def instantiate_template(self, replacements: dict[str, str], template: str, creator: Creator) -> str: # read content as binary to avoid platform specific line normalization (\n -> \r\n) binary = read_binary(self.__module__, template) text = binary.decode("utf-8", errors="strict") - for key, value in replacements.items(): - value_uni = self._repr_unicode(creator, value) - text = text.replace(key, self.quote(value_uni)) - return text + substitutions = self.substitutions(replacements, creator) + # one pass over the template, so a placeholder name inside an already quoted value (a path component named + # __VIRTUAL_NAME__) stays data instead of being rewritten by a later key from inside the finished literal + pattern = re.compile("|".join(re.escape(key) for key in sorted(substitutions, key=len, reverse=True))) + return pattern.sub(lambda match: substitutions[match.group()], text) + + def substitutions(self, replacements: dict[str, str], creator: Creator) -> dict[str, str]: + """Map each placeholder to the final text that replaces it in the template. + + :param replacements: the raw placeholder values + :param creator: the creator of the environment + + :returns: the placeholder to substituted text mapping + + """ + return {key: self.quote(self._repr_unicode(creator, value)) for key, value in replacements.items()} @staticmethod - def _repr_unicode(creator, value): # noqa: ARG004 + def _repr_unicode(creator: Creator, value: str) -> str: # ruff:ignore[unused-static-method-argument] return value # by default, we just let it be unicode diff --git a/src/virtualenv/activation/xonsh/__init__.py b/src/virtualenv/activation/xonsh/__init__.py new file mode 100644 index 000000000..e9784d89a --- /dev/null +++ b/src/virtualenv/activation/xonsh/__init__.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from virtualenv.activation.via_template import ViaTemplateActivator + +if TYPE_CHECKING: + from collections.abc import Iterator + from pathlib import Path + + from virtualenv.create.creator import Creator + + +class XonshActivator(ViaTemplateActivator): + def templates(self) -> Iterator[str]: + yield "activate.xsh" + + @staticmethod + def quote(string: str) -> str: + """Quote as a Python literal — xonsh parses the activation script as Python.""" + return repr(string) + + def replacements(self, creator: Creator, dest_folder: Path) -> dict[str, str]: + data = super().replacements(creator, dest_folder) + data.update({ + "__TCL_LIBRARY__": getattr(creator.interpreter, "tcl_lib", None) or "", + "__TK_LIBRARY__": getattr(creator.interpreter, "tk_lib", None) or "", + }) + return data + + +__all__ = [ + "XonshActivator", +] diff --git a/src/virtualenv/activation/xonsh/activate.xsh b/src/virtualenv/activation/xonsh/activate.xsh new file mode 100644 index 000000000..92f781d80 --- /dev/null +++ b/src/virtualenv/activation/xonsh/activate.xsh @@ -0,0 +1,101 @@ +# This file must be used with "source bin/activate.xsh" *from xonsh*. +# You cannot run it directly. + + +class _VirtualEnvActivator: + """xonsh activation for virtualenv. + + Rendered into ``/bin/activate.xsh`` by the template in + ``src/virtualenv/activation/xonsh/`` at generate-time. + + - Project: https://github.com/pypa/virtualenv + - Template source: https://github.com/pypa/virtualenv/tree/main/src/virtualenv/activation/xonsh + """ + + # Stashed in _OLD_VIRTUAL_{name} when the variable was not set before + # activation. deactivate treats this as "unset" rather than "restore to + # this string". Chosen to be vanishingly unlikely as a real value. + _UNSET_SENTINEL = "__virtualenv_was_not_set__" + + def __init__(self): + self.env = __xonsh__.env + try: + # Values substituted by virtualenv's XonshActivator at generate-time. + # If this file was sourced before template rendering, the bare + # identifiers below resolve to NameError. + self.embedded_virtual_env = __VIRTUAL_ENV__ + self.embedded_virtual_prompt = __VIRTUAL_PROMPT__ + self.embedded_bin_name = __BIN_NAME__ + self.embedded_tcl_library = __TCL_LIBRARY__ + self.embedded_tk_library = __TK_LIBRARY__ + except NameError: + print("virtualenv activate.xsh: template was not rendered") + raise + # Variables this activator may save+override; deactivate walks this list. + self.managed_vars = ("PATH", "PYTHONHOME", "TCL_LIBRARY", "TK_LIBRARY") + + def _backup_name(self, name): + return f"_OLD_VIRTUAL_{name}" + + def _save(self, name): + """Stash the current value (or sentinel if unset) so deactivate can undo.""" + backup = self._backup_name(name) + self.env[backup] = self.env[name] if name in self.env else self._UNSET_SENTINEL + + def _override(self, name, value): + self._save(name) + self.env[name] = value + + def _drop(self, name): + self._save(name) + self.env.pop(name, None) + + def register_pydoc(self): + aliases["pydoc"] = ["python", "-m", "pydoc"] + + def unregister_pydoc(self): + aliases.pop("pydoc", None) + + def activate(self): + from os.path import join, basename + + aliases["deactivate"] = self.deactivate + self.deactivate(["nondestructive"]) # wipe any stale state from a prior activation + + $VIRTUAL_ENV = self.embedded_virtual_env + $VIRTUAL_ENV_PROMPT = self.embedded_virtual_prompt or basename($VIRTUAL_ENV) + + self._override("PATH", [join($VIRTUAL_ENV, self.embedded_bin_name), *$PATH]) + self._drop("PYTHONHOME") + for name, value in (("TCL_LIBRARY", self.embedded_tcl_library), ("TK_LIBRARY", self.embedded_tk_library)): + if value: + self._override(name, value) + + self.register_pydoc() + + def deactivate(self, args=None): + for name in self.managed_vars: + backup = self._backup_name(name) + if backup not in self.env: + continue + previous = self.env[backup] + del self.env[backup] + if previous == self._UNSET_SENTINEL: + self.env.pop(name, None) + else: + self.env[name] = previous + for name in ("VIRTUAL_ENV", "VIRTUAL_ENV_PROMPT"): + self.env.pop(name, None) + self.unregister_pydoc() + if args is None or "nondestructive" not in args: + del aliases["deactivate"] + try: + del __xonsh__.xontrib.virtualenv + except AttributeError: + pass + + +if not hasattr(__xonsh__, "xontrib"): + __xonsh__.xontrib = __xonsh__.imp.types.SimpleNamespace() +__xonsh__.xontrib.virtualenv = _VirtualEnvActivator() +__xonsh__.xontrib.virtualenv.activate() diff --git a/src/virtualenv/app_data/__init__.py b/src/virtualenv/app_data/__init__.py index d7f148023..46416fb28 100644 --- a/src/virtualenv/app_data/__init__.py +++ b/src/virtualenv/app_data/__init__.py @@ -4,25 +4,49 @@ import logging import os +import shutil +from typing import TYPE_CHECKING, Any -from platformdirs import user_data_dir +from platformdirs import user_cache_dir, user_data_dir from .na import AppDataDisabled from .read_only import ReadOnlyAppData from .via_disk_folder import AppDataDiskFolder from .via_tempdir import TempAppData +if TYPE_CHECKING: + from collections.abc import Mapping + + from .base import AppData + LOGGER = logging.getLogger(__name__) -def _default_app_data_dir(env): +def _default_app_data_dir(env: Mapping[str, str]) -> str: key = "VIRTUALENV_OVERRIDE_APP_DATA" if key in env: return env[key] - return user_data_dir(appname="virtualenv", appauthor="pypa") + return _cache_dir_with_migration() + + +def _cache_dir_with_migration() -> str: + new_dir = user_cache_dir(appname="virtualenv", appauthor="pypa") + old_dir = user_data_dir(appname="virtualenv", appauthor="pypa") + if new_dir == old_dir: + return new_dir + if os.path.isdir(old_dir) and not os.path.isdir(new_dir): + LOGGER.info("migrating app data from %s to %s", old_dir, new_dir) + try: + shutil.move(old_dir, new_dir) + except OSError as exception: + LOGGER.warning( + "could not migrate app data from %s to %s: %r, using old location", old_dir, new_dir, exception + ) + return old_dir + return new_dir -def make_app_data(folder, **kwargs): +def make_app_data(folder: str | None, **kwargs: Any) -> AppData: # ruff:ignore[any-type] is_read_only = kwargs.pop("read_only") env = kwargs.pop("env") if kwargs: # py3+ kwonly @@ -36,12 +60,11 @@ def make_app_data(folder, **kwargs): if is_read_only: return ReadOnlyAppData(folder) - if not os.path.isdir(folder): - try: - os.makedirs(folder) - LOGGER.debug("created app data folder %s", folder) - except OSError as exception: - LOGGER.info("could not create app data folder %s due to %r", folder, exception) + try: + os.makedirs(folder, exist_ok=True) + LOGGER.debug("created app data folder %s", folder) + except OSError as exception: + LOGGER.info("could not create app data folder %s due to %r", folder, exception) if os.access(folder, os.W_OK): return AppDataDiskFolder(folder) diff --git a/src/virtualenv/app_data/base.py b/src/virtualenv/app_data/base.py index 2077deebd..6210c1cdf 100644 --- a/src/virtualenv/app_data/base.py +++ b/src/virtualenv/app_data/base.py @@ -4,52 +4,92 @@ from abc import ABC, abstractmethod from contextlib import contextmanager +from typing import TYPE_CHECKING from virtualenv.info import IS_ZIPAPP +if TYPE_CHECKING: + from collections.abc import Generator + from pathlib import Path + from typing import Any + class AppData(ABC): """Abstract storage interface for the virtualenv application.""" @abstractmethod - def close(self): + def close(self) -> None: """Called before virtualenv exits.""" @abstractmethod - def reset(self): + def reset(self) -> None: """Called when the user passes in the reset app data.""" @abstractmethod - def py_info(self, path): + def py_info(self, path: Path) -> ContentStore: + """Return a content store for cached interpreter information at the given path. + + :param path: the interpreter executable path + + :returns: a content store for the cached data + + """ raise NotImplementedError @abstractmethod - def py_info_clear(self): + def py_info_clear(self) -> None: + """Clear all cached interpreter information.""" raise NotImplementedError @property - def can_update(self): + def can_update(self) -> bool: + """``True`` if this app data store supports updating cached content.""" raise NotImplementedError @abstractmethod - def embed_update_log(self, distribution, for_py_version): + def embed_update_log(self, distribution: str, for_py_version: str) -> ContentStore: + """Return a content store for the embed update log of a distribution. + + :param distribution: the package name (e.g. ``pip``) + :param for_py_version: the target Python version string + + :returns: a content store for the update log + + """ raise NotImplementedError @property - def house(self): + def house(self) -> Path: + """The root directory of the application data store.""" raise NotImplementedError @property - def transient(self): + def transient(self) -> bool: + """``True`` if this app data store is transient and does not persist across runs.""" raise NotImplementedError @abstractmethod - def wheel_image(self, for_py_version, name): + def wheel_image(self, for_py_version: str, name: str) -> Path: + """Return the path to a cached wheel image. + + :param for_py_version: the target Python version string + :param name: the package name + + :returns: the path to the cached wheel + + """ raise NotImplementedError @contextmanager - def ensure_extracted(self, path, to_folder=None): - """Some paths might be within the zipapp, unzip these to a path on the disk.""" + def ensure_extracted(self, path: Path, to_folder: Path | None = None) -> Generator[Path]: + """Ensure a path is available on disk, extracting from zipapp if needed. + + :param path: the path to ensure is available + :param to_folder: optional target directory for extraction + + :returns: yields the usable path on disk + + """ if IS_ZIPAPP: with self.extract(path, to_folder) as result: yield result @@ -58,36 +98,67 @@ def ensure_extracted(self, path, to_folder=None): @abstractmethod @contextmanager - def extract(self, path, to_folder): + def extract(self, path: Path, to_folder: Path | None) -> Generator[Path]: + """Extract a path from the zipapp to a location on disk. + + :param path: the path to extract + :param to_folder: optional target directory + + :returns: yields the extracted path + + """ raise NotImplementedError @abstractmethod @contextmanager - def locked(self, path): + def locked(self, path: Path) -> Generator[None]: + """Acquire an exclusive lock on the given path. + + :param path: the path to lock + + """ raise NotImplementedError class ContentStore(ABC): + """A store for reading and writing cached content.""" + @abstractmethod - def exists(self): + def exists(self) -> bool: + """Check if the stored content exists. + + :returns: ``True`` if content exists + + """ raise NotImplementedError @abstractmethod - def read(self): + def read(self) -> Any: # ruff:ignore[any-type] + """Read the stored content. + + :returns: the stored content + + """ raise NotImplementedError @abstractmethod - def write(self, content): + def write(self, content: Any) -> None: # ruff:ignore[any-type] + """Write content to the store. + + :param content: the content to write + + """ raise NotImplementedError @abstractmethod - def remove(self): + def remove(self) -> None: + """Remove the stored content.""" raise NotImplementedError @abstractmethod @contextmanager - def locked(self): - pass + def locked(self) -> Generator[None]: + """Acquire an exclusive lock on this content store.""" __all__ = [ diff --git a/src/virtualenv/app_data/na.py b/src/virtualenv/app_data/na.py index 921e83a81..c2079dafb 100644 --- a/src/virtualenv/app_data/na.py +++ b/src/virtualenv/app_data/na.py @@ -1,9 +1,15 @@ from __future__ import annotations from contextlib import contextmanager +from typing import TYPE_CHECKING from .base import AppData, ContentStore +if TYPE_CHECKING: + from collections.abc import Generator + from pathlib import Path + from typing import Any, NoReturn + class AppDataDisabled(AppData): """No application cache available (most likely as we don't have write permissions).""" @@ -16,53 +22,53 @@ def __init__(self) -> None: error = RuntimeError("no app data folder available, probably no write access to the folder") - def close(self): + def close(self) -> None: """Do nothing.""" - def reset(self): + def reset(self) -> None: """Do nothing.""" - def py_info(self, path): # noqa: ARG002 + def py_info(self, path: Path) -> ContentStoreNA: # ruff:ignore[unused-method-argument] return ContentStoreNA() - def embed_update_log(self, distribution, for_py_version): # noqa: ARG002 + def embed_update_log(self, distribution: str, for_py_version: str) -> ContentStoreNA: # ruff:ignore[unused-method-argument] return ContentStoreNA() - def extract(self, path, to_folder): # noqa: ARG002 + def extract(self, path: Path, to_folder: Path | None) -> NoReturn: # ruff:ignore[unused-method-argument] raise self.error @contextmanager - def locked(self, path): # noqa: ARG002 + def locked(self, path: Path) -> Generator[None]: # ruff:ignore[unused-method-argument] """Do nothing.""" yield @property - def house(self): + def house(self) -> NoReturn: raise self.error - def wheel_image(self, for_py_version, name): # noqa: ARG002 + def wheel_image(self, for_py_version: str, name: str) -> NoReturn: # ruff:ignore[unused-method-argument] raise self.error - def py_info_clear(self): + def py_info_clear(self) -> None: """Nothing to clear.""" class ContentStoreNA(ContentStore): - def exists(self): + def exists(self) -> bool: return False - def read(self): + def read(self) -> None: """Nothing to read.""" return - def write(self, content): + def write(self, content: Any) -> None: # ruff:ignore[any-type] """Nothing to write.""" - def remove(self): + def remove(self) -> None: """Nothing to remove.""" @contextmanager - def locked(self): + def locked(self) -> Generator[None]: yield diff --git a/src/virtualenv/app_data/read_only.py b/src/virtualenv/app_data/read_only.py index 952dbad5a..6e92e2f5f 100644 --- a/src/virtualenv/app_data/read_only.py +++ b/src/virtualenv/app_data/read_only.py @@ -1,11 +1,16 @@ from __future__ import annotations import os.path +from typing import TYPE_CHECKING from virtualenv.util.lock import NoOpFileLock from .via_disk_folder import AppDataDiskFolder, PyInfoStoreDisk +if TYPE_CHECKING: + from pathlib import Path + from typing import NoReturn + class ReadOnlyAppData(AppDataDiskFolder): can_update = False @@ -24,15 +29,15 @@ def reset(self) -> None: def py_info_clear(self) -> None: raise NotImplementedError - def py_info(self, path): + def py_info(self, path: Path) -> _PyInfoStoreDiskReadOnly: return _PyInfoStoreDiskReadOnly(self.py_info_at, path) - def embed_update_log(self, distribution, for_py_version): + def embed_update_log(self, distribution: str, for_py_version: str) -> NoReturn: raise NotImplementedError class _PyInfoStoreDiskReadOnly(PyInfoStoreDisk): - def write(self, content): # noqa: ARG002 + def write(self, content: str) -> NoReturn: # ruff:ignore[unused-method-argument] msg = "read-only app data python info cannot be updated" raise RuntimeError(msg) diff --git a/src/virtualenv/app_data/via_disk_folder.py b/src/virtualenv/app_data/via_disk_folder.py index 9ebe91c2e..f5e2ccd08 100644 --- a/src/virtualenv/app_data/via_disk_folder.py +++ b/src/virtualenv/app_data/via_disk_folder.py @@ -1,26 +1,28 @@ -""" -A rough layout of the current storage goes as: - -virtualenv-app-data -├── py - -│ └── *.json/lock -├── wheel -│ ├── house -│ │ └── *.whl -│ └── -> 3.9 -│ ├── img- -│ │ └── image -│ │ └── -> CopyPipInstall / SymlinkPipInstall -│ │ └── -> pip-20.1.1-py2.py3-none-any -│ └── embed -│ └── 3 -> json format versioning -│ └── *.json -> for every distribution contains data about newer embed versions and releases -└─── unzip - └── - ├── py_info.py - ├── debug.py - └── _virtualenv.py -""" # noqa: D415 +r"""A rough layout of the current storage goes as: + +:: + + virtualenv-app-data + ├── py - + │ └── *.json/lock + ├── wheel + │ ├── house + │ │ └── *.whl + │ └── -> 3.9 + │ ├── img- + │ │ └── image + │ │ └── -> CopyPipInstall / SymlinkPipInstall + │ │ └── -> pip-20.1.1-py2.py3-none-any + │ └── embed + │ └── 3 -> json format versioning + │ └── *.json -> for every distribution contains data about newer embed versions and releases + └─── unzip + └── + ├── py_info.py + ├── debug.py + └── _virtualenv.py + +""" # ruff:ignore[missing-terminal-punctuation] from __future__ import annotations @@ -29,6 +31,7 @@ from abc import ABC from contextlib import contextmanager, suppress from hashlib import sha256 +from typing import TYPE_CHECKING, Any from virtualenv.util.lock import ReentrantFileLock from virtualenv.util.path import safe_delete @@ -37,6 +40,10 @@ from .base import AppData, ContentStore +if TYPE_CHECKING: + from collections.abc import Generator + from pathlib import Path + LOGGER = logging.getLogger(__name__) @@ -46,7 +53,7 @@ class AppDataDiskFolder(AppData): transient = False can_update = True - def __init__(self, folder) -> None: + def __init__(self, folder: str) -> None: self.lock = ReentrantFileLock(folder) def __repr__(self) -> str: @@ -55,22 +62,22 @@ def __repr__(self) -> str: def __str__(self) -> str: return str(self.lock.path) - def reset(self): + def reset(self) -> None: LOGGER.debug("reset app data folder %s", self.lock.path) safe_delete(self.lock.path) - def close(self): + def close(self) -> None: """Do nothing.""" @contextmanager - def locked(self, path): - path_lock = self.lock / path + def locked(self, path: Path) -> Generator[None]: + path_lock = self.lock / path # ty: ignore[unsupported-operator] with path_lock: yield path_lock.path @contextmanager - def extract(self, path, to_folder): - root = ReentrantFileLock(to_folder()) if to_folder is not None else self.lock / "unzip" / __version__ + def extract(self, path: Path, to_folder: Path | None) -> Generator[Path]: + root = ReentrantFileLock(to_folder()) if to_folder is not None else self.lock / "unzip" / __version__ # ty: ignore[call-non-callable] with root.lock_for_key(path.name): dest = root.path / path.name if not dest.exists(): @@ -78,13 +85,13 @@ def extract(self, path, to_folder): yield dest @property - def py_info_at(self): - return self.lock / "py_info" / "2" + def py_info_at(self) -> ReentrantFileLock: + return self.lock / "py_info" / "5" # ty: ignore[invalid-return-type] - def py_info(self, path): + def py_info(self, path: Path) -> PyInfoStoreDisk: return PyInfoStoreDisk(self.py_info_at, path) - def py_info_clear(self): + def py_info_clear(self) -> None: """clear py info.""" py_info_folder = self.py_info_at with py_info_folder: @@ -94,39 +101,39 @@ def py_info_clear(self): if filename.exists(): filename.unlink() - def embed_update_log(self, distribution, for_py_version): - return EmbedDistributionUpdateStoreDisk(self.lock / "wheel" / for_py_version / "embed" / "3", distribution) + def embed_update_log(self, distribution: str, for_py_version: str) -> EmbedDistributionUpdateStoreDisk: + return EmbedDistributionUpdateStoreDisk(self.lock / "wheel" / for_py_version / "embed" / "3", distribution) # ty: ignore[invalid-argument-type] @property - def house(self): + def house(self) -> Path: path = self.lock.path / "wheel" / "house" path.mkdir(parents=True, exist_ok=True) return path - def wheel_image(self, for_py_version, name): + def wheel_image(self, for_py_version: str, name: str) -> Path: return self.lock.path / "wheel" / for_py_version / "image" / "1" / name class JSONStoreDisk(ContentStore, ABC): - def __init__(self, in_folder, key, msg_args) -> None: + def __init__(self, in_folder: ReentrantFileLock, key: str, msg_args: tuple[str, ...]) -> None: self.in_folder = in_folder self.key = key self.msg_args = (*msg_args, self.file) @property - def file(self): + def file(self) -> Path: return self.in_folder.path / f"{self.key}.json" - def exists(self): + def exists(self) -> bool: return self.file.exists() - def read(self): + def read(self) -> Any: # ruff:ignore[any-type] data, bad_format = None, False try: data = json.loads(self.file.read_text(encoding="utf-8")) except ValueError: bad_format = True - except Exception: # noqa: BLE001, S110 + except Exception: # ruff:ignore[blind-except, try-except-pass] pass else: LOGGER.debug("got %s %s from %s", *self.msg_args) @@ -136,16 +143,16 @@ def read(self): self.remove() return None - def remove(self): + def remove(self) -> None: self.file.unlink() LOGGER.debug("removed %s %s at %s", *self.msg_args) @contextmanager - def locked(self): + def locked(self) -> Generator[None]: with self.in_folder.lock_for_key(self.key): yield - def write(self, content): + def write(self, content: Any) -> None: # ruff:ignore[any-type] folder = self.file.parent folder.mkdir(parents=True, exist_ok=True) self.file.write_text(json.dumps(content, sort_keys=True, indent=2), encoding="utf-8") @@ -153,13 +160,13 @@ def write(self, content): class PyInfoStoreDisk(JSONStoreDisk): - def __init__(self, in_folder, path) -> None: + def __init__(self, in_folder: ReentrantFileLock, path: Path) -> None: key = sha256(str(path).encode("utf-8")).hexdigest() - super().__init__(in_folder, key, ("python info of", path)) + super().__init__(in_folder, key, ("python info of", path)) # ty: ignore[invalid-argument-type] class EmbedDistributionUpdateStoreDisk(JSONStoreDisk): - def __init__(self, in_folder, distribution) -> None: + def __init__(self, in_folder: ReentrantFileLock, distribution: str) -> None: super().__init__( in_folder, distribution, diff --git a/src/virtualenv/app_data/via_tempdir.py b/src/virtualenv/app_data/via_tempdir.py index 884a570ce..a690f3c79 100644 --- a/src/virtualenv/app_data/via_tempdir.py +++ b/src/virtualenv/app_data/via_tempdir.py @@ -2,11 +2,15 @@ import logging from tempfile import mkdtemp +from typing import TYPE_CHECKING from virtualenv.util.path import safe_delete from .via_disk_folder import AppDataDiskFolder +if TYPE_CHECKING: + from typing import NoReturn + LOGGER = logging.getLogger(__name__) @@ -18,14 +22,14 @@ def __init__(self) -> None: super().__init__(folder=mkdtemp()) LOGGER.debug("created temporary app data folder %s", self.lock.path) - def reset(self): + def reset(self) -> None: """This is a temporary folder, is already empty to start with.""" - def close(self): + def close(self) -> None: LOGGER.debug("remove temporary app data folder %s", self.lock.path) safe_delete(self.lock.path) - def embed_update_log(self, distribution, for_py_version): + def embed_update_log(self, distribution: str, for_py_version: str) -> NoReturn: raise NotImplementedError diff --git a/src/virtualenv/config/cli/parser.py b/src/virtualenv/config/cli/parser.py index fd1dc0956..f97a80675 100644 --- a/src/virtualenv/config/cli/parser.py +++ b/src/virtualenv/config/cli/parser.py @@ -1,8 +1,14 @@ from __future__ import annotations import os +import shutil from argparse import SUPPRESS, ArgumentDefaultsHelpFormatter, ArgumentParser, Namespace from collections import OrderedDict +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from argparse import Action + from collections.abc import Mapping, Sequence from virtualenv.config.convert import get_type from virtualenv.config.env_var import get_env_var @@ -10,27 +16,46 @@ class VirtualEnvOptions(Namespace): - def __init__(self, **kwargs) -> None: + def __init__(self, **kwargs: Any) -> None: # ruff:ignore[any-type] super().__init__(**kwargs) - self._src = None - self._sources = {} + self._src: str | None = None + self._sources: dict[str, str] = {} + + def set_src(self, key: str, value: Any, src: str) -> None: # ruff:ignore[any-type] + """Set an option value and record where it came from. + + :param key: the option name + :param value: the option value + :param src: the source of the value (e.g. ``"cli"``, ``"env var"``, ``"default"``) - def set_src(self, key, value, src): + """ setattr(self, key, value) if src.startswith("env var"): src = "env var" self._sources[key] = src - def __setattr__(self, key, value) -> None: - if getattr(self, "_src", None) is not None: - self._sources[key] = self._src + def __setattr__(self, key: str, value: Any) -> None: # ruff:ignore[any-type] + if (src := getattr(self, "_src", None)) is not None: + self._sources[key] = src super().__setattr__(key, value) - def get_source(self, key): + def get_source(self, key: str) -> str | None: + """Return the source that provided a given option value. + + :param key: the option name + + :returns: the source string (e.g. ``"cli"``, ``"env var"``, ``"default"``), or ``None`` if not tracked + + """ return self._sources.get(key) @property - def verbosity(self): + def verbosity(self) -> int | None: + """The verbosity level, computed as ``verbose - quiet``, clamped to zero. + + :returns: the verbosity level, or ``None`` if neither ``--verbose`` nor ``--quiet`` has been parsed yet + + """ if not hasattr(self, "verbose") and not hasattr(self, "quiet"): return None return max(self.verbose - self.quiet, 0) @@ -42,7 +67,13 @@ def __repr__(self) -> str: class VirtualEnvConfigParser(ArgumentParser): """Custom option parser which updates its defaults by checking the configuration files and environmental vars.""" - def __init__(self, options=None, env=None, *args, **kwargs) -> None: + def __init__( + self, + options: VirtualEnvOptions | None = None, + env: Mapping[str, str] | None = None, + *args: Any, # ruff:ignore[any-type] + **kwargs: Any, # ruff:ignore[any-type] + ) -> None: env = os.environ if env is None else env self.file_config = IniConfig(env) self.epilog_list = [] @@ -60,14 +91,14 @@ def __init__(self, options=None, env=None, *args, **kwargs) -> None: self._interpreter = None self._app_data = None - def _fix_defaults(self): + def _fix_defaults(self) -> None: for action in self._actions: action_id = id(action) if action_id not in self._fixed: self._fix_default(action) self._fixed.add(action_id) - def _fix_default(self, action): + def _fix_default(self, action: Action) -> None: if hasattr(action, "default") and hasattr(action, "dest") and action.default != SUPPRESS: as_type = get_type(action) names = OrderedDict((i.lstrip("-").replace("-", "_"), None) for i in action.option_strings) @@ -81,44 +112,60 @@ def _fix_default(self, action): outcome = self.file_config.get(name, as_type) if outcome is not None: break + if tri_state := action.default is None and isinstance(action.const, bool): + outcome = _as_passed_flag(action.const, outcome) if outcome is not None: - action.default, action.default_source = outcome + action.default, default_source = outcome + vars(action)["default_source"] = default_source + elif tri_state and hasattr(self.options, action.dest): + return # a sibling flag sharing this destination already set it else: outcome = action.default, "default" self.options.set_src(action.dest, *outcome) - def enable_help(self): + def enable_help(self) -> None: self._fix_defaults() self.add_argument("-h", "--help", action="help", default=SUPPRESS, help="show this help message and exit") - def parse_known_args(self, args=None, namespace=None): + def parse_known_args( # ty: ignore[invalid-method-override] + self, args: Sequence[str] | None = None, namespace: VirtualEnvOptions | None = None + ) -> tuple[VirtualEnvOptions, list[str]]: if namespace is None: namespace = self.options elif namespace is not self.options: msg = "can only pass in parser.options" raise ValueError(msg) self._fix_defaults() - self.options._src = "cli" # noqa: SLF001 + self.options._src = "cli" # ruff:ignore[private-member-access] try: namespace.env = self.env return super().parse_known_args(args, namespace=namespace) finally: - self.options._src = None # noqa: SLF001 + self.options._src = None # ruff:ignore[private-member-access] class HelpFormatter(ArgumentDefaultsHelpFormatter): - def __init__(self, prog, **kwargs) -> None: - super().__init__(prog, max_help_position=32, width=240, **kwargs) + def __init__(self, prog: str, **kwargs: Any) -> None: # ruff:ignore[any-type] + super().__init__(prog, max_help_position=32, width=shutil.get_terminal_size().columns, **kwargs) - def _get_help_string(self, action): + def _get_help_string(self, action: Action) -> str | None: text = super()._get_help_string(action) - if hasattr(action, "default_source"): + if text is not None and hasattr(action, "default_source"): default = " (default: %(default)s)" if text.endswith(default): text = f"{text[: -len(default)]} (default: %(default)s -> from %(default_source)s)" return text +def _as_passed_flag(const: bool, outcome: tuple[bool, str] | None) -> tuple[bool, str] | None: + # an env var or ini value for a --x/--no-x pair means "as if the flag was passed": true stores the flag's value, while + # false on the negative flag leaves the choice unset rather than flipping it on + if outcome is None or const: + return outcome + value, source = outcome + return (False, source) if value else None + + __all__ = [ "HelpFormatter", "VirtualEnvConfigParser", diff --git a/src/virtualenv/config/convert.py b/src/virtualenv/config/convert.py index ef7581dbd..cee2db41e 100644 --- a/src/virtualenv/config/convert.py +++ b/src/virtualenv/config/convert.py @@ -2,20 +2,24 @@ import logging import os -from typing import ClassVar +from typing import TYPE_CHECKING, ClassVar + +if TYPE_CHECKING: + from argparse import Action + from typing import Any LOGGER = logging.getLogger(__name__) class TypeData: - def __init__(self, default_type, as_type) -> None: + def __init__(self, default_type: type, as_type: type) -> None: self.default_type = default_type self.as_type = as_type def __repr__(self) -> str: return f"{self.__class__.__name__}(base={self.default_type}, as={self.as_type})" - def convert(self, value): + def convert(self, value: str) -> Any: # ruff:ignore[any-type] return self.default_type(value) @@ -31,7 +35,7 @@ class BoolType(TypeData): "off": False, } - def convert(self, value): + def convert(self, value: str) -> bool: if value.lower() not in self.BOOLEAN_STATES: msg = f"Not a boolean: {value}" raise ValueError(msg) @@ -39,17 +43,17 @@ def convert(self, value): class NoneType(TypeData): - def convert(self, value): + def convert(self, value: str) -> str | None: if not value: return None return str(value) class ListType(TypeData): - def _validate(self): + def _validate(self) -> None: """no op.""" - def convert(self, value, flatten=True): # noqa: ARG002, FBT002 + def convert(self, value: str | list[str], flatten: bool = True) -> list[Any]: # ruff:ignore[unused-method-argument, boolean-default-value-positional-argument] values = self.split_values(value) result = [] for a_value in values: @@ -57,28 +61,24 @@ def convert(self, value, flatten=True): # noqa: ARG002, FBT002 result.extend(sub_values) return [self.as_type(i) for i in result] - def split_values(self, value): - """ - Split the provided value into a list. + def split_values(self, value: str | bytes | list[str]) -> list[str]: + """Split the provided value into a list. + + First this is done by newlines. If there were no newlines in the text, then we next try to split by comma. - First this is done by newlines. If there were no newlines in the text, - then we next try to split by comma. """ - if isinstance(value, (str, bytes)): - # Use `splitlines` rather than a custom check for whether there is - # more than one line. This ensures that the full `splitlines()` - # logic is supported here. + if isinstance(value, bytes): + value = value.decode() + if isinstance(value, str): + # `splitlines` rather than a manual newline check, so its full line-boundary logic applies. values = value.splitlines() if len(values) <= 1: values = value.split(",") - values = filter(None, [x.strip() for x in values]) - else: - values = list(value) - - return values + return [stripped for x in values if (stripped := x.strip())] + return list(value) -def convert(value, as_type, source): +def convert(value: str, as_type: TypeData, source: str) -> Any: # ruff:ignore[any-type] """Convert the value as a given type where the value comes from the given source.""" try: return as_type.convert(value) @@ -90,10 +90,11 @@ def convert(value, as_type, source): _CONVERT = {bool: BoolType, type(None): NoneType, list: ListType} -def get_type(action): - default_type = type(action.default) +def get_type(action: Action) -> TypeData: + # a flag that stores a bool over an unset default is tri-state, so its env var and ini values read as bools + default_type = bool if action.default is None and isinstance(action.const, bool) else type(action.default) as_type = default_type if action.type is None else action.type - return _CONVERT.get(default_type, TypeData)(default_type, as_type) + return _CONVERT.get(default_type, TypeData)(default_type, as_type) # ty: ignore[invalid-argument-type] __all__ = [ diff --git a/src/virtualenv/config/env_var.py b/src/virtualenv/config/env_var.py index e12723471..498e3db9c 100644 --- a/src/virtualenv/config/env_var.py +++ b/src/virtualenv/config/env_var.py @@ -1,18 +1,26 @@ from __future__ import annotations from contextlib import suppress +from typing import TYPE_CHECKING from .convert import convert +if TYPE_CHECKING: + from collections.abc import Mapping + from typing import Any -def get_env_var(key, as_type, env): - """ - Get the environment variable option. + from .convert import TypeData + + +def get_env_var(key: str, as_type: TypeData, env: Mapping[str, str]) -> tuple[Any, str] | None: + """Get the environment variable option. :param key: the config key requested :param as_type: the type we would like to convert it to :param env: environment variables to use - :return: + + :returns: the converted value and source, or None if not set + """ environ_key = f"VIRTUALENV_{key.upper()}" if env.get(environ_key): diff --git a/src/virtualenv/config/ini.py b/src/virtualenv/config/ini.py index ed0a1b930..e60a4f831 100644 --- a/src/virtualenv/config/ini.py +++ b/src/virtualenv/config/ini.py @@ -2,15 +2,21 @@ import logging import os -from configparser import ConfigParser +from configparser import ConfigParser, Error from pathlib import Path -from typing import ClassVar +from typing import TYPE_CHECKING, ClassVar, Final from platformdirs import user_config_dir from .convert import convert -LOGGER = logging.getLogger(__name__) +if TYPE_CHECKING: + from collections.abc import Mapping + from typing import Any + + from .convert import TypeData + +_LOGGER: Final[logging.Logger] = logging.getLogger(__name__) class IniConfig: @@ -19,7 +25,7 @@ class IniConfig: section = "virtualenv" - def __init__(self, env=None) -> None: + def __init__(self, env: Mapping[str, str] | None = None) -> None: env = os.environ if env is None else env config_file = env.get(self.VIRTUALENV_CONFIG_FILE_ENV_VAR, None) self.is_env_var = config_file is not None @@ -30,29 +36,27 @@ def __init__(self, env=None) -> None: self.config_file = config_file self._cache = {} + self.has_config_file: bool | None = None + self.has_virtualenv_section = False + self.config_parser: Final[ConfigParser] = ConfigParser() exception = None - self.has_config_file = None try: self.has_config_file = self.config_file.exists() - except OSError as exc: - exception = exc - else: if self.has_config_file: self.config_file = self.config_file.resolve() - self.config_parser = ConfigParser() - try: - self._load() - self.has_virtualenv_section = self.config_parser.has_section(self.section) - except Exception as exc: # noqa: BLE001 - exception = exc + self._load() + self.has_virtualenv_section = self.config_parser.has_section(self.section) + except (OSError, UnicodeError, Error) as exc: + self.has_config_file = None + exception = exc if exception is not None: - LOGGER.error("failed to read config file %s because %r", config_file, exception) + _LOGGER.error("failed to read config file %s because %r", config_file, exception) - def _load(self): - with self.config_file.open("rt", encoding="utf-8") as file_handler: + def _load(self) -> None: + with self.config_file.open("rt", encoding="utf-8-sig") as file_handler: return self.config_parser.read_file(file_handler) - def get(self, key, as_type): + def get(self, key: str, as_type: TypeData) -> tuple[Any, str] | None: cache_key = key, as_type if cache_key in self._cache: return self._cache[cache_key] @@ -61,7 +65,7 @@ def get(self, key, as_type): raw_value = self.config_parser.get(self.section, key.lower()) value = convert(raw_value, as_type, source) result = value, source - except Exception: # noqa: BLE001 + except Exception: # ruff:ignore[blind-except] result = None self._cache[cache_key] = result return result @@ -70,8 +74,11 @@ def __bool__(self) -> bool: return bool(self.has_config_file) and bool(self.has_virtualenv_section) @property - def epilog(self): + def epilog(self) -> str: return ( f"\nconfig file {self.config_file} {self.STATE[self.has_config_file]} " f"(change{'d' if self.is_env_var else ''} via env var {self.VIRTUALENV_CONFIG_FILE_ENV_VAR})" ) + + +__all__ = ["IniConfig"] diff --git a/src/virtualenv/create/creator.py b/src/virtualenv/create/creator.py index 1e577ada5..25878957b 100644 --- a/src/virtualenv/create/creator.py +++ b/src/virtualenv/create/creator.py @@ -10,10 +10,21 @@ from ast import literal_eval from collections import OrderedDict from pathlib import Path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from argparse import ArgumentParser + from typing import Any, NoReturn + + from python_discovery import PythonInfo + + from virtualenv.app_data.base import AppData + from virtualenv.config.cli.parser import VirtualEnvOptions + +from os.path import commonpath -from virtualenv.discovery.cached_py_info import LogCmd from virtualenv.util.path import safe_delete -from virtualenv.util.subprocess import run_cmd +from virtualenv.util.subprocess import LogCmd, run_cmd from virtualenv.version import __version__ from .pyenv_cfg import PyEnvCfg @@ -31,52 +42,85 @@ def __init__(self) -> None: class Creator(ABC): """A class that given a python Interpreter creates a virtual environment.""" - def __init__(self, options, interpreter) -> None: - """ - Construct a new virtual environment creator. + def __init__(self, options: VirtualEnvOptions, interpreter: PythonInfo) -> None: + """Construct a new virtual environment creator. :param options: the CLI option as parsed from :meth:`add_parser_arguments` :param interpreter: the interpreter to create virtual environment from + """ self.interpreter = interpreter self._debug = None self.dest = Path(options.dest) self.clear = options.clear self.no_vcs_ignore = options.no_vcs_ignore + self.venv_redirect = options.venv_redirect self.pyenv_cfg = PyEnvCfg.from_folder(self.dest) self.app_data = options.app_data self.env = options.env + self.prompt = getattr(options, "prompt", None) + + if TYPE_CHECKING: + + @property + def exe(self) -> Path: ... + + @property + def env_name(self) -> str: ... + + @property + def bin_dir(self) -> Path: ... + + @property + def script_dir(self) -> Path: ... + + @property + def libs(self) -> list[Path]: ... + + @property + def purelib(self) -> Path: ... + + @property + def platlib(self) -> Path: ... def __repr__(self) -> str: return f"{self.__class__.__name__}({', '.join(f'{k}={v}' for k, v in self._args())})" - def _args(self): + def _args(self) -> list[tuple[str, Any]]: return [ ("dest", str(self.dest)), ("clear", self.clear), ("no_vcs_ignore", self.no_vcs_ignore), + ("venv_redirect", self.venv_redirect), ] @classmethod - def can_create(cls, interpreter): # noqa: ARG003 - """ - Determine if we can create a virtual environment. + def can_create(cls, interpreter: PythonInfo) -> CreatorMeta | bool | None: # ruff:ignore[unused-class-method-argument] + """Determine if we can create a virtual environment. :param interpreter: the interpreter in question - :return: ``None`` if we can't create, any other object otherwise that will be forwarded to \ - :meth:`add_parser_arguments` + + :returns: ``None`` if we can't create, any other object otherwise that will be forwarded to + :meth:`add_parser_arguments` + """ return True @classmethod - def add_parser_arguments(cls, parser, interpreter, meta, app_data): # noqa: ARG003 - """ - Add CLI arguments for the creator. + def add_parser_arguments( + cls, + parser: ArgumentParser, + interpreter: PythonInfo, # ruff:ignore[unused-class-method-argument] + meta: CreatorMeta, # ruff:ignore[unused-class-method-argument] + app_data: AppData, # ruff:ignore[unused-class-method-argument] + ) -> None: + """Add CLI arguments for the creator. :param parser: the CLI parser :param app_data: the application data folder :param interpreter: the interpreter we're asked to create virtual environment for :param meta: value as returned by :meth:`can_create` + """ parser.add_argument( "dest", @@ -97,18 +141,35 @@ def add_parser_arguments(cls, parser, interpreter, meta, app_data): # noqa: ARG help="don't create VCS ignore directive in the destination directory", default=False, ) + parser.add_argument( + "--venv-redirect", + dest="venv_redirect", + action="store_const", + const=True, + help="point a PEP-832 .venv redirect file next to the destination at the created environment, even outside " + "a project folder; unset, virtualenv does so only beside a pyproject.toml with no .venv yet", + default=None, + ) + parser.add_argument( + "--no-venv-redirect", + dest="venv_redirect", + action="store_const", + const=False, + help="write no PEP-832 .venv redirect file", + default=None, + ) @abstractmethod - def create(self): + def create(self) -> None: """Perform the virtual environment creation.""" raise NotImplementedError @classmethod - def validate_dest(cls, raw_value): # noqa: C901 + def validate_dest(cls, raw_value: str) -> str: # ruff:ignore[complex-structure] """No path separator in the path, valid chars and must be write-able.""" - def non_write_able(dest, value): - common = Path(*os.path.commonprefix([value.parts, dest.parts])) + def non_write_able(dest: Path, value: Path) -> NoReturn: + common = Path(commonpath([str(value), str(dest)])) msg = f"the destination {dest.relative_to(common)} is not write-able at {common}" raise ArgumentTypeError(msg) @@ -122,7 +183,7 @@ def non_write_able(dest, value): trip = char.encode(encoding, **kwargs).decode(encoding) if trip == char: continue - raise ValueError(trip) # noqa: TRY301 + raise ValueError(trip) # ruff:ignore[raise-within-try] except ValueError: refused[char] = None if refused: @@ -153,7 +214,7 @@ def non_write_able(dest, value): dest = base return str(value) - def run(self): + def run(self) -> None: if self.dest.exists() and self.clear: LOGGER.debug("delete %s", self.dest) safe_delete(self.dest) @@ -162,8 +223,10 @@ def run(self): self.set_pyenv_cfg() if not self.no_vcs_ignore: self.setup_ignore_vcs() + if self.venv_redirect is not False: + self.point_venv_redirect() - def add_cachedir_tag(self): + def add_cachedir_tag(self) -> None: """Generate a file indicating that this is not meant to be backed up.""" cachedir_tag_file = self.dest / "CACHEDIR.TAG" if not cachedir_tag_file.exists(): @@ -175,14 +238,24 @@ def add_cachedir_tag(self): """).strip() cachedir_tag_file.write_text(cachedir_tag_text, encoding="utf-8") - def set_pyenv_cfg(self): + def set_pyenv_cfg(self) -> None: self.pyenv_cfg.content = OrderedDict() - self.pyenv_cfg["home"] = os.path.dirname(os.path.abspath(self.interpreter.system_executable)) + system_executable = self.interpreter.system_executable or self.interpreter.executable + assert system_executable is not None # ruff:ignore[assert] + self.pyenv_cfg["home"] = os.path.dirname(os.path.abspath(system_executable)) self.pyenv_cfg["implementation"] = self.interpreter.implementation - self.pyenv_cfg["version_info"] = ".".join(str(i) for i in self.interpreter.version_info) + version_info = self.interpreter.version_info + self.pyenv_cfg["python-version"] = f"{version_info.major}.{version_info.minor}" + self.pyenv_cfg["version_info"] = ".".join(str(i) for i in version_info) + self.pyenv_cfg["version"] = ".".join(str(i) for i in version_info[:3]) + self.pyenv_cfg["executable"] = os.path.realpath(system_executable) + self.pyenv_cfg["command"] = f"{sys.executable} -m virtualenv {self.dest}" self.pyenv_cfg["virtualenv"] = __version__ + if self.prompt is not None: + prompt_value = os.path.basename(os.getcwd()) if self.prompt == "." else self.prompt + self.pyenv_cfg["prompt"] = prompt_value - def setup_ignore_vcs(self): + def setup_ignore_vcs(self) -> None: """Generate ignore instructions for version control systems.""" # mark this folder to be ignored by VCS, handle https://www.python.org/dev/peps/pep-0610/#registered-vcs git_ignore = self.dest / ".gitignore" @@ -194,19 +267,47 @@ def setup_ignore_vcs(self): # Bazaar - does not support ignore files in sub-directories, only at root level via .bzrignore # Subversion - does not support ignore files, requires direct manipulation with the svn tool + def point_venv_redirect(self) -> None: + """Point the ``.venv`` redirect file of the parent folder at the environment, per PEP-832 (provisional).""" + if self.dest.name == ".venv": # the environment is the .venv itself, no redirect needed + return + redirect = self.dest.parent / ".venv" + if self.venv_redirect is None and ( + redirect.exists() or redirect.is_symlink() or not (self.dest.parent / "pyproject.toml").is_file() + ): # unset: claim the default only for a project folder that has none yet + return + if redirect.is_symlink() or redirect.is_dir(): + LOGGER.debug("%s keeps being the default environment", redirect) + return + if redirect.exists() and not _points_at_virtualenv(redirect): + LOGGER.debug("%s points at an environment virtualenv did not create, leaving it", redirect) + return + try: + redirect.write_text(f"{self.dest.name}\n", encoding="utf-8") + except OSError as exc: + LOGGER.warning("could not point %s at %s - %s", redirect, self.dest, exc) + @property - def debug(self): - """:return: debug information about the virtual environment (only valid after :meth:`create` has run)""" + def debug(self) -> dict[str, Any] | None: + """Debug information about the virtual environment (only valid after :meth:`create` has run).""" if self._debug is None and self.exe is not None: self._debug = get_env_debug_info(self.exe, self.debug_script(), self.app_data, self.env) return self._debug @staticmethod - def debug_script(): + def debug_script() -> Path: return DEBUG_SCRIPT -def get_env_debug_info(env_exe, debug_script, app_data, env): +def _points_at_virtualenv(redirect: Path) -> bool: + try: + line = redirect.read_text(encoding="utf-8").removesuffix("\n") # universal newlines already turned \r\n into \n + return bool(line) and "virtualenv" in PyEnvCfg.from_folder(redirect.parent / line) + except (OSError, UnicodeDecodeError): # unreadable, so not a redirect virtualenv can claim + return False + + +def get_env_debug_info(env_exe: Path, debug_script: Path, app_data: AppData, env: dict[str, str]) -> dict[str, Any]: env = env.copy() env.pop("PYTHONPATH", None) @@ -216,25 +317,29 @@ def get_env_debug_info(env_exe, debug_script, app_data, env): code, out, err = run_cmd(cmd) try: - if code != 0: - if out: - result = literal_eval(out) - else: - if code == 2 and "file" in err: # noqa: PLR2004 - # Re-raise FileNotFoundError from `run_cmd()` - raise OSError(err) # noqa: TRY301 - raise Exception(err) # noqa: TRY002, TRY301 - else: - result = json.loads(out) - if err: - result["err"] = err - except Exception as exception: # noqa: BLE001 + result = _parse_debug_output(code, out, err) + except Exception as exception: # ruff:ignore[blind-except] return {"out": out, "err": err, "returncode": code, "exception": repr(exception)} if "sys" in result and "path" in result["sys"]: del result["sys"]["path"][0] return result +def _parse_debug_output(code: int, out: str, err: str) -> dict[str, Any]: + if code != 0: + if out: + result = literal_eval(out) + elif code == 2 and "file" in err: # ruff:ignore[magic-value-comparison] + raise OSError(err) + else: + raise Exception(err) # ruff:ignore[raise-vanilla-class] + else: + result = json.loads(out) + if err: + result["err"] = err + return result + + __all__ = [ "Creator", "CreatorMeta", diff --git a/src/virtualenv/create/debug.py b/src/virtualenv/create/debug.py index 8a4845e98..938c434d7 100644 --- a/src/virtualenv/create/debug.py +++ b/src/virtualenv/create/debug.py @@ -5,7 +5,7 @@ import sys # built-in -def encode_path(value): +def encode_path(value: object) -> str | None: if value is None: return None if not isinstance(value, (str, bytes)): @@ -15,25 +15,19 @@ def encode_path(value): return value -def encode_list_path(value): +def encode_list_path(value: list[object]) -> list[str | None]: return [encode_path(i) for i in value] -def run(): +def run() -> None: """Print debug data about the virtual environment.""" - try: - from collections import OrderedDict # noqa: PLC0415 - except ImportError: # pragma: no cover - # this is possible if the standard library cannot be accessed - - OrderedDict = dict # pragma: no cover # noqa: N806 - result = OrderedDict([("sys", OrderedDict())]) + sys_info: dict[str, str | list[str | None] | None] = {} + result: dict[str, str | dict[str, str | list[str | None] | None] | None] = {"sys": sys_info} path_keys = ( "executable", "_base_executable", "prefix", "base_prefix", - "real_prefix", "exec_prefix", "base_exec_prefix", "path", @@ -42,40 +36,38 @@ def run(): for key in path_keys: value = getattr(sys, key, None) value = encode_list_path(value) if isinstance(value, list) else encode_path(value) - result["sys"][key] = value - result["sys"]["fs_encoding"] = sys.getfilesystemencoding() - result["sys"]["io_encoding"] = getattr(sys.stdout, "encoding", None) + sys_info[key] = value + sys_info["fs_encoding"] = sys.getfilesystemencoding() + sys_info["io_encoding"] = getattr(sys.stdout, "encoding", None) result["version"] = sys.version try: - import sysconfig # noqa: PLC0415 + import sysconfig # ruff:ignore[import-outside-top-level] - # https://bugs.python.org/issue22199 - makefile = getattr(sysconfig, "get_makefile_filename", getattr(sysconfig, "_get_makefile_filename", None)) - result["makefile_filename"] = encode_path(makefile()) + result["makefile_filename"] = encode_path(sysconfig.get_makefile_filename()) except ImportError: pass - import os # landmark # noqa: PLC0415 + import os # landmark # ruff:ignore[import-outside-top-level] result["os"] = repr(os) try: - import site # site # noqa: PLC0415 + import site # site # ruff:ignore[import-outside-top-level] result["site"] = repr(site) except ImportError as exception: # pragma: no cover result["site"] = repr(exception) # pragma: no cover try: - import datetime # site # noqa: PLC0415 + import datetime as dt # site # ruff:ignore[import-outside-top-level] - result["datetime"] = repr(datetime) + result["datetime"] = repr(dt) except ImportError as exception: # pragma: no cover result["datetime"] = repr(exception) # pragma: no cover try: - import math # site # noqa: PLC0415 + import math # site # ruff:ignore[import-outside-top-level] result["math"] = repr(math) except ImportError as exception: # pragma: no cover @@ -83,7 +75,7 @@ def run(): # try to print out, this will validate if other core modules are available (json in this case) try: - import json # noqa: PLC0415 + import json # ruff:ignore[import-outside-top-level] result["json"] = repr(json) except ImportError as exception: @@ -95,7 +87,7 @@ def run(): except (ValueError, TypeError) as exception: # pragma: no cover sys.stderr.write(repr(exception)) sys.stdout.write(repr(result)) # pragma: no cover - raise SystemExit(1) # noqa: B904 # pragma: no cover + raise SystemExit(1) # ruff:ignore[raise-without-from-inside-except] # pragma: no cover if __name__ == "__main__": diff --git a/src/virtualenv/create/describe.py b/src/virtualenv/create/describe.py index 1ee250cbc..7b5a1399f 100644 --- a/src/virtualenv/create/describe.py +++ b/src/virtualenv/create/describe.py @@ -3,16 +3,22 @@ from abc import ABC from collections import OrderedDict from pathlib import Path +from typing import TYPE_CHECKING from virtualenv.info import IS_WIN +if TYPE_CHECKING: + from typing import Any + + from python_discovery import PythonInfo + class Describe: """Given a host interpreter tell us information about what the created interpreter might look like.""" suffix = ".exe" if IS_WIN else "" - def __init__(self, dest, interpreter) -> None: + def __init__(self, dest: Path, interpreter: PythonInfo) -> None: self.interpreter = interpreter self.dest = dest self._stdlib = None @@ -21,84 +27,84 @@ def __init__(self, dest, interpreter) -> None: self._conf_vars = None @property - def bin_dir(self): + def bin_dir(self) -> Path: return self.script_dir @property - def script_dir(self): + def script_dir(self) -> Path: return self.dest / self.interpreter.install_path("scripts") @property - def purelib(self): + def purelib(self) -> Path: return self.dest / self.interpreter.install_path("purelib") @property - def platlib(self): + def platlib(self) -> Path: return self.dest / self.interpreter.install_path("platlib") @property - def libs(self): + def libs(self) -> list[Path]: return list(OrderedDict(((self.platlib, None), (self.purelib, None))).keys()) @property - def stdlib(self): + def stdlib(self) -> Path: if self._stdlib is None: self._stdlib = Path(self.interpreter.sysconfig_path("stdlib", config_var=self._config_vars)) return self._stdlib @property - def stdlib_platform(self): + def stdlib_platform(self) -> Path: if self._stdlib_platform is None: self._stdlib_platform = Path(self.interpreter.sysconfig_path("platstdlib", config_var=self._config_vars)) return self._stdlib_platform @property - def _config_vars(self): + def _config_vars(self) -> dict[str, Any]: if self._conf_vars is None: self._conf_vars = self._calc_config_vars(self.dest) return self._conf_vars - def _calc_config_vars(self, to): + def _calc_config_vars(self, to: Path) -> dict[str, Any]: sys_vars = self.interpreter.sysconfig_vars - return {k: (to if v is not None and v.startswith(self.interpreter.prefix) else v) for k, v in sys_vars.items()} + if (prefix := self.interpreter.prefix) is None: # no prefix means no value can live under it + return dict(sys_vars) + return {k: (to if isinstance(v, str) and v.startswith(prefix) else v) for k, v in sys_vars.items()} @classmethod - def can_describe(cls, interpreter): # noqa: ARG003 + def can_describe(cls, interpreter: PythonInfo) -> bool: # ruff:ignore[unused-class-method-argument] """Knows means it knows how the output will look.""" return True @property - def env_name(self): + def env_name(self) -> str: return self.dest.parts[-1] @property - def exe(self): + def exe(self) -> Path: return self.bin_dir / f"{self.exe_stem()}{self.suffix}" @classmethod - def exe_stem(cls): + def exe_stem(cls) -> str: """Executable name without suffix - there seems to be no standard way to get this without creating it.""" raise NotImplementedError - def script(self, name): + def script(self, name: str) -> Path: return self.script_dir / f"{name}{self.suffix}" class Python3Supports(Describe, ABC): - @classmethod - def can_describe(cls, interpreter): - return interpreter.version_info.major == 3 and super().can_describe(interpreter) # noqa: PLR2004 + pass class PosixSupports(Describe, ABC): @classmethod - def can_describe(cls, interpreter): + def can_describe(cls, interpreter: PythonInfo) -> bool: return interpreter.os == "posix" and super().can_describe(interpreter) class WindowsSupports(Describe, ABC): @classmethod - def can_describe(cls, interpreter): + def can_describe(cls, interpreter: PythonInfo) -> bool: return interpreter.os == "nt" and super().can_describe(interpreter) diff --git a/src/virtualenv/create/pyenv_cfg.py b/src/virtualenv/create/pyenv_cfg.py index 1d1beacff..805e9c45f 100644 --- a/src/virtualenv/create/pyenv_cfg.py +++ b/src/virtualenv/create/pyenv_cfg.py @@ -3,59 +3,84 @@ import logging import os from collections import OrderedDict +from typing import TYPE_CHECKING, Final + +from virtualenv.util.text import collapse_line_boundaries + +if TYPE_CHECKING: + from pathlib import Path LOGGER = logging.getLogger(__name__) +# a plain value such as 3.14 or true would otherwise turn into a path whenever the working directory has an entry so named +_PATH_KEYS: Final[frozenset[str]] = frozenset({ + "home", + "executable", + "base-prefix", + "base-exec-prefix", + "base-executable", + "venvlauncher_command", +}) + class PyEnvCfg: - def __init__(self, content, path) -> None: + def __init__(self, content: OrderedDict[str, str], path: Path) -> None: self.content = content self.path = path @classmethod - def from_folder(cls, folder): + def from_folder(cls, folder: Path) -> PyEnvCfg: return cls.from_file(folder / "pyvenv.cfg") @classmethod - def from_file(cls, path): + def from_file(cls, path: Path) -> PyEnvCfg: content = cls._read_values(path) if path.exists() else OrderedDict() return PyEnvCfg(content, path) @staticmethod - def _read_values(path): + def _read_values(path: Path) -> OrderedDict[str, str]: content = OrderedDict() for line in path.read_text(encoding="utf-8").splitlines(): - equals_at = line.index("=") - key = line[:equals_at].strip() - value = line[equals_at + 1 :].strip() + key, separator, value = line.partition("=") + if not separator or (key := key.strip()).startswith("#"): + continue + value = value.strip() + if len(value) > 1 and value[0] in {"'", '"'} and value[0] == value[-1]: + value = value[1:-1] content[key] = value return content - def write(self): + def write(self) -> None: LOGGER.debug("write %s", self.path) text = "" for key, value in self.content.items(): - normalized_value = os.path.realpath(value) if value and os.path.exists(value) else value - line = f"{key} = {normalized_value}" + if key == "prompt" and value: + normalized_value = f'"{value}"' + elif key in _PATH_KEYS and os.path.exists(value): + # abspath rather than realpath keeps symlinked prefixes intact, as venv does (#2770) + normalized_value = os.path.abspath(value) + else: + normalized_value = value + line = f"{collapse_line_boundaries(key)} = {collapse_line_boundaries(normalized_value)}" LOGGER.debug("\t%s", line) text += line text += "\n" self.path.write_text(text, encoding="utf-8") - def refresh(self): + def refresh(self) -> OrderedDict[str, str]: self.content = self._read_values(self.path) return self.content - def __setitem__(self, key, value) -> None: + def __setitem__(self, key: str, value: str) -> None: self.content[key] = value - def __getitem__(self, key): + def __getitem__(self, key: str) -> str: return self.content[key] - def __contains__(self, item) -> bool: + def __contains__(self, item: str) -> bool: return item in self.content - def update(self, other): + def update(self, other: dict[str, str]) -> PyEnvCfg: self.content.update(other) return self diff --git a/src/virtualenv/create/via_global_ref/_virtualenv.py b/src/virtualenv/create/via_global_ref/_virtualenv.py index 0d95b28e0..c5b5fa686 100644 --- a/src/virtualenv/create/via_global_ref/_virtualenv.py +++ b/src/virtualenv/create/via_global_ref/_virtualenv.py @@ -5,21 +5,26 @@ import contextlib import os import sys +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + import types + from collections.abc import Callable + from importlib.machinery import ModuleSpec VIRTUALENV_PATCH_FILE = os.path.abspath(__file__) -def patch_dist(dist): - """ - Distutils allows user to configure some arguments via a configuration file: - https://docs.python.org/3/install/index.html#distutils-configuration-files. +def patch_dist(dist: types.ModuleType) -> None: + """Distutils allows user to configure some arguments via a configuration file: https://docs.python.org/3/install/index.html#distutils-configuration-files. Some of this arguments though don't make sense in context of the virtual environment files, let's fix them up. - """ # noqa: D205 + + """ # we cannot allow some install config as that would get packages installed outside of the virtual environment old_parse_config_files = dist.Distribution.parse_config_files - def parse_config_files(self, *args, **kwargs): + def parse_config_files(self, *args: object, **kwargs: object) -> object: # ruff:ignore[missing-type-function-argument] result = old_parse_config_files(self, *args, **kwargs) install = self.get_option_dict("install") @@ -48,9 +53,9 @@ class _Finder: # lock[0] is threading.Lock(), but initialized lazily to avoid importing threading very early at startup, # because there are gevent-based applications that need to be first to import threading by themselves. # See https://github.com/pypa/virtualenv/issues/1895 for details. - lock = [] # noqa: RUF012 + lock = [] # ruff:ignore[mutable-class-default] - def find_spec(self, fullname, path, target=None): # noqa: ARG002 + def find_spec(self, fullname: str, path: object, target: object = None) -> ModuleSpec | None: # ruff:ignore[unused-method-argument] # Guard against race conditions during file rewrite by checking if _DISTUTILS_PATCH is defined. # This can happen when the file is being overwritten while it's being imported by another process. # See https://github.com/pypa/virtualenv/issues/2969 for details. @@ -58,10 +63,10 @@ def find_spec(self, fullname, path, target=None): # noqa: ARG002 distutils_patch = _DISTUTILS_PATCH except NameError: return None - if fullname in distutils_patch and self.fullname is None: # noqa: PLR1702 + if fullname in distutils_patch and self.fullname is None: # initialize lock[0] lazily if len(self.lock) == 0: - import threading # noqa: PLC0415 + import threading # ruff:ignore[import-outside-top-level] lock = threading.Lock() # there is possibility that two threads T1 and T2 are simultaneously running into find_spec, @@ -71,53 +76,39 @@ def find_spec(self, fullname, path, target=None): # noqa: ARG002 # https://docs.python.org/3/faq/library.html#what-kinds-of-global-value-mutation-are-thread-safe self.lock.append(lock) - from functools import partial # noqa: PLC0415 - from importlib.util import find_spec # noqa: PLC0415 + from functools import partial # ruff:ignore[import-outside-top-level] + from importlib.util import find_spec # ruff:ignore[import-outside-top-level] with self.lock[0]: self.fullname = fullname + spec = None try: - spec = find_spec(fullname, path) - if spec is not None: - # https://www.python.org/dev/peps/pep-0451/#how-loading-will-work - is_new_api = hasattr(spec.loader, "exec_module") - func_name = "exec_module" if is_new_api else "load_module" - old = getattr(spec.loader, func_name) - func = self.exec_module if is_new_api else self.load_module - if old is not func: - try: # noqa: SIM105 - setattr(spec.loader, func_name, partial(func, old)) - except AttributeError: - pass # C-Extension loaders are r/o such as zipimporter with <3.7 - return spec + spec = find_spec(fullname, path) # ty: ignore[invalid-argument-type] finally: self.fullname = None + if spec is not None: + return self._patch_spec(spec, partial) return None + def _patch_spec(self, spec: ModuleSpec, partial: Callable[..., object]) -> ModuleSpec: + old = getattr(spec.loader, "exec_module", None) + if old is not None and old is not self.exec_module: + try: # ruff:ignore[suppressible-exception] + spec.loader.exec_module = partial(self.exec_module, old) # ty: ignore[invalid-assignment] + except AttributeError: + pass + return spec + @staticmethod - def exec_module(old, module): + def exec_module(old: Callable[..., object], module: types.ModuleType) -> None: old(module) try: distutils_patch = _DISTUTILS_PATCH except NameError: return if module.__name__ in distutils_patch: - # patch_dist or its dependencies may not be defined during file rewrite - with contextlib.suppress(NameError): - patch_dist(module) - - @staticmethod - def load_module(old, name): - module = old(name) - try: - distutils_patch = _DISTUTILS_PATCH - except NameError: - return module - if module.__name__ in distutils_patch: - # patch_dist or its dependencies may not be defined during file rewrite with contextlib.suppress(NameError): patch_dist(module) - return module sys.meta_path.insert(0, _Finder()) diff --git a/src/virtualenv/create/via_global_ref/api.py b/src/virtualenv/create/via_global_ref/api.py index 3f04f4653..b3e01f11e 100644 --- a/src/virtualenv/create/via_global_ref/api.py +++ b/src/virtualenv/create/via_global_ref/api.py @@ -1,13 +1,22 @@ from __future__ import annotations import logging -import os from abc import ABC from pathlib import Path +from typing import TYPE_CHECKING from virtualenv.create.creator import Creator, CreatorMeta from virtualenv.info import fs_supports_symlink +if TYPE_CHECKING: + from argparse import ArgumentParser + from typing import Any + + from python_discovery import PythonInfo + + from virtualenv.app_data.base import AppData + from virtualenv.config.cli.parser import VirtualEnvOptions + LOGGER = logging.getLogger(__name__) @@ -20,22 +29,30 @@ def __init__(self) -> None: self.symlink_error = "the filesystem does not supports symlink" @property - def can_copy(self): + def can_copy(self) -> bool: return not self.copy_error @property - def can_symlink(self): + def can_symlink(self) -> bool: return not self.symlink_error class ViaGlobalRefApi(Creator, ABC): - def __init__(self, options, interpreter) -> None: + def __init__(self, options: VirtualEnvOptions, interpreter: PythonInfo) -> None: super().__init__(options, interpreter) self.symlinks = self._should_symlink(options) self.enable_system_site_package = options.system_site + if TYPE_CHECKING: + + @property + def purelib(self) -> Path: ... + + @property + def script_dir(self) -> Path: ... + @staticmethod - def _should_symlink(options): + def _should_symlink(options: VirtualEnvOptions) -> bool: # Priority of where the option is set to follow the order: CLI, env var, file, hardcoded. # If both set at same level prefers copy over symlink. copies, symlinks = getattr(options, "copies", False), getattr(options, "symlinks", False) @@ -52,7 +69,9 @@ def _should_symlink(options): return False # fallback to copy @classmethod - def add_parser_arguments(cls, parser, interpreter, meta, app_data): + def add_parser_arguments( + cls, parser: ArgumentParser, interpreter: PythonInfo, meta: ViaGlobalRefMeta, app_data: AppData + ) -> None: # ty: ignore[invalid-method-override] super().add_parser_arguments(parser, interpreter, meta, app_data) parser.add_argument( "--system-site-packages", @@ -88,10 +107,15 @@ def add_parser_arguments(cls, parser, interpreter, meta, app_data): help="try to use copies rather than symlinks, even when symlinks are the default for the platform", ) - def create(self): + def create(self) -> None: self.install_patch() - def install_patch(self): + def install_patch(self) -> None: + # Python 3.10+ ignores the distutils install config keys this patch guards against (pip does so by default, + # setuptools and CPython's vendored distutils drop them too), so the runtime import hook only earns its + # startup cost on 3.9. See https://github.com/pypa/virtualenv/issues/3181. + if self.interpreter.version_info >= (3, 10): + return text = self.env_patch_text() if text: pth = self.purelib / "_virtualenv.pth" @@ -101,16 +125,15 @@ def install_patch(self): LOGGER.debug("create %s", dest_path) dest_path.write_text(text, encoding="utf-8") - def env_patch_text(self): + def env_patch_text(self) -> str: """Patch the distutils package to not be derailed by its configuration files.""" with self.app_data.ensure_extracted(Path(__file__).parent / "_virtualenv.py") as resolved_path: - text = resolved_path.read_text(encoding="utf-8") - return text.replace('"__SCRIPT_DIR__"', repr(os.path.relpath(str(self.script_dir), str(self.purelib)))) + return resolved_path.read_text(encoding="utf-8") - def _args(self): + def _args(self) -> list[tuple[str, Any]]: return [*super()._args(), ("global", self.enable_system_site_package)] - def set_pyenv_cfg(self): + def set_pyenv_cfg(self) -> None: super().set_pyenv_cfg() self.pyenv_cfg["include-system-site-packages"] = "true" if self.enable_system_site_package else "false" diff --git a/src/virtualenv/create/via_global_ref/builtin/builtin_way.py b/src/virtualenv/create/via_global_ref/builtin/builtin_way.py index 791b1d93d..005c23640 100644 --- a/src/virtualenv/create/via_global_ref/builtin/builtin_way.py +++ b/src/virtualenv/create/via_global_ref/builtin/builtin_way.py @@ -1,15 +1,21 @@ from __future__ import annotations from abc import ABC +from typing import TYPE_CHECKING from virtualenv.create.creator import Creator from virtualenv.create.describe import Describe +if TYPE_CHECKING: + from python_discovery import PythonInfo + + from virtualenv.config.cli.parser import VirtualEnvOptions + class VirtualenvBuiltin(Creator, Describe, ABC): """A creator that does operations itself without delegation, if we can create it we can also describe it.""" - def __init__(self, options, interpreter) -> None: + def __init__(self, options: VirtualEnvOptions, interpreter: PythonInfo) -> None: Creator.__init__(self, options, interpreter) Describe.__init__(self, self.dest, interpreter) diff --git a/src/virtualenv/create/via_global_ref/builtin/cpython/common.py b/src/virtualenv/create/via_global_ref/builtin/cpython/common.py index 5cc993bda..0a88fad75 100644 --- a/src/virtualenv/create/via_global_ref/builtin/cpython/common.py +++ b/src/virtualenv/create/via_global_ref/builtin/cpython/common.py @@ -4,19 +4,25 @@ from abc import ABC from collections import OrderedDict from pathlib import Path +from typing import TYPE_CHECKING from virtualenv.create.describe import PosixSupports, WindowsSupports from virtualenv.create.via_global_ref.builtin.ref import RefMust, RefWhen from virtualenv.create.via_global_ref.builtin.via_global_self_do import ViaGlobalRefVirtualenvBuiltin +if TYPE_CHECKING: + from collections.abc import Generator + + from python_discovery import PythonInfo + class CPython(ViaGlobalRefVirtualenvBuiltin, ABC): @classmethod - def can_describe(cls, interpreter): + def can_describe(cls, interpreter: PythonInfo) -> bool: return interpreter.implementation == "CPython" and super().can_describe(interpreter) @classmethod - def exe_stem(cls): + def exe_stem(cls) -> str: return "python" @@ -24,41 +30,59 @@ class CPythonPosix(CPython, PosixSupports, ABC): """Create a CPython virtual environment on POSIX platforms.""" @classmethod - def _executables(cls, interpreter): - host_exe = Path(interpreter.system_executable) - major, minor = interpreter.version_info.major, interpreter.version_info.minor - targets = OrderedDict((i, None) for i in ["python", f"python{major}", f"python{major}.{minor}", host_exe.name]) + def _executables(cls, interpreter: PythonInfo) -> Generator[tuple[Path, list[str], str, str]]: + host_exe = Path(interpreter.system_exe) + minor = interpreter.version_info.minor + names = [ + "python", + "python3", + f"python3.{minor}", + *((f"python3.{minor}t",) if interpreter.free_threaded else ()), + host_exe.name, + ] + targets = OrderedDict((i, None) for i in names) yield host_exe, list(targets.keys()), RefMust.NA, RefWhen.ANY class CPythonWindows(CPython, WindowsSupports, ABC): @classmethod - def _executables(cls, interpreter): + def _executables(cls, interpreter: PythonInfo) -> Generator[tuple[Path, list[str], str, str]]: # symlink of the python executables does not work reliably, copy always instead # - https://bugs.python.org/issue42013 # - venv host = cls.host_python(interpreter) - names = {"python.exe", host.name} - if interpreter.version_info.major == 3: # noqa: PLR2004 - names.update({"python3.exe", "python3"}) + minor = interpreter.version_info.minor + names = { + "python.exe", + "python3.exe", + "python3", + # host is the venvlauncher shim on CPython 3.13+, so the alias comes from the interpreter, not from host + Path(interpreter.system_exe).name, + *((f"python3.{minor}t.exe",) if interpreter.free_threaded else ()), + } for path in (host.parent / n for n in names): yield host, [path.name], RefMust.COPY, RefWhen.ANY # for more info on pythonw.exe see https://stackoverflow.com/a/30313091 python_w = host.parent / "pythonw.exe" - yield python_w, [python_w.name], RefMust.COPY, RefWhen.ANY + yield ( + python_w, + [python_w.name, "pythonw3.exe", *((f"pythonw3.{minor}t.exe",) if interpreter.free_threaded else ())], + RefMust.COPY, + RefWhen.ANY, + ) @classmethod - def host_python(cls, interpreter): - return Path(interpreter.system_executable) + def host_python(cls, interpreter: PythonInfo) -> Path: + return Path(interpreter.system_exe) -def is_mac_os_framework(interpreter): +def is_mac_os_framework(interpreter: PythonInfo) -> bool: if interpreter.platform == "darwin": return interpreter.sysconfig_vars.get("PYTHONFRAMEWORK") == "Python3" return False -def is_macos_brew(interpreter): +def is_macos_brew(interpreter: PythonInfo) -> bool: return interpreter.platform == "darwin" and _BREW.fullmatch(interpreter.system_prefix) is not None diff --git a/src/virtualenv/create/via_global_ref/builtin/cpython/cpython3.py b/src/virtualenv/create/via_global_ref/builtin/cpython/cpython3.py index a46da45dd..8543811f0 100644 --- a/src/virtualenv/create/via_global_ref/builtin/cpython/cpython3.py +++ b/src/virtualenv/create/via_global_ref/builtin/cpython/cpython3.py @@ -2,17 +2,27 @@ import abc import fnmatch -from itertools import chain from operator import methodcaller as method from pathlib import Path -from textwrap import dedent +from typing import TYPE_CHECKING from virtualenv.create.describe import Python3Supports -from virtualenv.create.via_global_ref.builtin.ref import ExePathRefToDest, PathRefToDest +from virtualenv.create.via_global_ref.builtin.ref import ExePathRefToDest, PathRefToDest, RefWhen from virtualenv.create.via_global_ref.store import is_store_python +from virtualenv.util.path import copy as copy_path +from virtualenv.util.path import ensure_dir from .common import CPython, CPythonPosix, CPythonWindows, is_mac_os_framework, is_macos_brew +if TYPE_CHECKING: + from collections.abc import Generator + + from python_discovery import PythonInfo + + from virtualenv.create.via_global_ref.builtin.ref import PathRef + from virtualenv.create.via_global_ref.builtin.via_global_self_do import BuiltinViaGlobalRefMeta + from virtualenv.create.via_global_ref.venv import Venv + class CPython3(CPython, Python3Supports, abc.ABC): """CPython 3 or later.""" @@ -20,62 +30,76 @@ class CPython3(CPython, Python3Supports, abc.ABC): class CPython3Posix(CPythonPosix, CPython3): @classmethod - def can_describe(cls, interpreter): + def can_describe(cls, interpreter: PythonInfo) -> bool: return ( is_mac_os_framework(interpreter) is False and is_macos_brew(interpreter) is False and super().can_describe(interpreter) ) - def env_patch_text(self): - text = super().env_patch_text() - if self.pyvenv_launch_patch_active(self.interpreter): - text += dedent( - """ - # for https://github.com/python/cpython/pull/9516, see https://github.com/pypa/virtualenv/issues/1704 - import os - if "__PYVENV_LAUNCHER__" in os.environ: - del os.environ["__PYVENV_LAUNCHER__"] - """, - ) - return text + @classmethod + def sources(cls, interpreter: PythonInfo) -> Generator[PathRef]: # ty: ignore[invalid-method-override] + yield from super().sources(interpreter) + if shared_lib := cls._shared_libpython(interpreter): + yield PathRefToDest(shared_lib, dest=cls._to_lib, when=RefWhen.COPY) + + @classmethod + def _to_lib(cls, creator: CPython3Posix, src: Path) -> Path: + return creator.dest / "lib" / src.name @classmethod - def pyvenv_launch_patch_active(cls, interpreter): - ver = interpreter.version_info - return interpreter.platform == "darwin" and ((3, 7, 8) > ver >= (3, 7) or (3, 8, 3) > ver >= (3, 8)) + def _shared_libpython(cls, interpreter: PythonInfo) -> Path | None: + if not interpreter.sysconfig_vars.get("Py_ENABLE_SHARED"): + return None + if not isinstance(instsoname := interpreter.sysconfig_vars.get("INSTSONAME"), str) or not instsoname: + return None + if not isinstance(libdir := interpreter.sysconfig_vars.get("LIBDIR"), str) or not libdir: + return None + if not (lib_path := Path(libdir) / instsoname).exists(): + return None + return lib_path + + def install_venv_shared_libs(self, venv_creator: Venv) -> None: + if venv_creator.symlinks: + return + if not (shared_lib := self._shared_libpython(venv_creator.interpreter)): + return + dest = venv_creator.dest / "lib" / shared_lib.name + ensure_dir(dest.parent) + copy_path(shared_lib, dest) class CPython3Windows(CPythonWindows, CPython3): """CPython 3 on Windows.""" @classmethod - def setup_meta(cls, interpreter): + def setup_meta(cls, interpreter: PythonInfo) -> BuiltinViaGlobalRefMeta | None: # ty: ignore[invalid-method-override] if is_store_python(interpreter): # store python is not supported here return None return super().setup_meta(interpreter) @classmethod - def sources(cls, interpreter): - if cls.has_shim(interpreter): - refs = cls.executables(interpreter) - else: - refs = chain( - cls.executables(interpreter), - cls.dll_and_pyd(interpreter), - cls.python_zip(interpreter), - ) - yield from refs + def sources(cls, interpreter: PythonInfo) -> Generator[PathRef]: # ty: ignore[invalid-method-override] + yield from cls.executables(interpreter) + if not cls.has_shim(interpreter): + yield from cls.dll_and_pyd(interpreter) + yield from cls.python_zip(interpreter) + + @classmethod + def _debug_suffix(cls, interpreter: PythonInfo) -> str: + return "_d" if getattr(interpreter, "debug_build", False) else "" @classmethod - def executables(cls, interpreter): + def executables(cls, interpreter: PythonInfo) -> list[PathRef] | Generator[PathRef]: sources = super().sources(interpreter) if interpreter.version_info >= (3, 13): - # Create new refs with corrected launcher paths - updated_sources = [] + host = cls.host_python(interpreter) + t_suffix = "t" if interpreter.free_threaded else "" + d_suffix = cls._debug_suffix(interpreter) + updated_sources: list[PathRef] = [] for ref in sources: - if ref.src.name == "python.exe": - launcher_path = ref.src.with_name("venvlauncher.exe") + if ref.base == "python.exe": + launcher_path = host.with_name(f"venvlauncher{t_suffix}{d_suffix}.exe") if launcher_path.exists(): new_ref = ExePathRefToDest( launcher_path, dest=ref.dest, targets=[ref.base, *ref.aliases], must=ref.must, when=ref.when @@ -83,7 +107,7 @@ def executables(cls, interpreter): updated_sources.append(new_ref) continue elif ref.src.name == "pythonw.exe": - w_launcher_path = ref.src.with_name("venvwlauncher.exe") + w_launcher_path = ref.src.with_name(f"venvwlauncher{t_suffix}{d_suffix}.exe") if w_launcher_path.exists(): new_ref = ExePathRefToDest( w_launcher_path, @@ -94,37 +118,39 @@ def executables(cls, interpreter): ) updated_sources.append(new_ref) continue - # Keep the original ref unchanged updated_sources.append(ref) return updated_sources return sources @classmethod - def has_shim(cls, interpreter): - return interpreter.version_info.minor >= 7 and cls.shim(interpreter) is not None # noqa: PLR2004 + def has_shim(cls, interpreter: PythonInfo) -> bool: + return cls.shim(interpreter) is not None @classmethod - def shim(cls, interpreter): + def shim(cls, interpreter: PythonInfo) -> Path | None: root = Path(interpreter.system_stdlib) / "venv" / "scripts" / "nt" - # Before 3.13 the launcher was called python.exe, after is venvlauncher.exe - # https://github.com/python/cpython/issues/112984 - exe_name = "venvlauncher.exe" if interpreter.version_info >= (3, 13) else "python.exe" - shim = root / exe_name - if shim.exists(): + d_suffix = cls._debug_suffix(interpreter) + if interpreter.version_info >= (3, 13): + # https://github.com/python/cpython/issues/112984 + t_suffix = "t" if interpreter.free_threaded else "" + exe_name = f"venvlauncher{t_suffix}{d_suffix}.exe" + else: + exe_name = f"python{d_suffix}.exe" + if (shim := root / exe_name).exists(): return shim return None @classmethod - def host_python(cls, interpreter): + def host_python(cls, interpreter: PythonInfo) -> Path: if cls.has_shim(interpreter): # starting with CPython 3.7 Windows ships with a venvlauncher.exe that avoids the need for dll/pyd copies # it also means the wrapper must be copied to avoid bugs such as https://bugs.python.org/issue42013 - return cls.shim(interpreter) + return cls.shim(interpreter) # ty: ignore[invalid-return-type] return super().host_python(interpreter) @classmethod - def dll_and_pyd(cls, interpreter): - folders = [Path(interpreter.system_executable).parent] + def dll_and_pyd(cls, interpreter: PythonInfo) -> Generator[PathRefToDest]: + folders = [Path(interpreter.system_exe).parent] # May be missing on some Python hosts. # See https://github.com/pypa/virtualenv/issues/2368 @@ -135,28 +161,37 @@ def dll_and_pyd(cls, interpreter): for folder in folders: for file in folder.iterdir(): if file.suffix in {".pyd", ".dll"}: + # Skip pywin32 DLLs to avoid conflicts with pywin32 installation + # pywin32 has its own post-install that places DLLs in site-packages/pywin32_system32 + # See https://github.com/pypa/virtualenv/issues/2662 + if cls._is_pywin32_dll(file.name): + continue yield PathRefToDest(file, cls.to_bin) @classmethod - def python_zip(cls, interpreter): + def _is_pywin32_dll(cls, filename: str) -> bool: + """Check if a DLL file belongs to pywin32.""" + # pywin32 DLLs follow patterns like: pywintypes39.dll, pythoncom39.dll + name_lower = filename.lower() + return name_lower.startswith(("pywintypes", "pythoncom")) + + @classmethod + def python_zip(cls, interpreter: PythonInfo) -> Generator[PathRefToDest]: + """``python{VERSION}.zip`` contains compiled ``*.pyc`` std lib packages, where ``VERSION`` is ``py_version_nodot`` var from the ``sysconfig`` module. + + See https://docs.python.org/3/using/windows.html#the-embeddable-package, ``discovery.py_info.PythonInfo`` class + (interpreter), and ``python -m sysconfig`` output. + + The embeddable Python distribution for Windows includes ``python{VERSION}.zip`` and ``python{VERSION}._pth`` + files. User can move/rename the zip file and edit ``sys.path`` by editing the ``_pth`` file. Here the + ``pattern`` is used only for the default zip file name. + """ - "python{VERSION}.zip" contains compiled *.pyc std lib packages, where - "VERSION" is `py_version_nodot` var from the `sysconfig` module. - :see: https://docs.python.org/3/using/windows.html#the-embeddable-package - :see: `discovery.py_info.PythonInfo` class (interpreter). - :see: `python -m sysconfig` output. - - :note: The embeddable Python distribution for Windows includes - "python{VERSION}.zip" and "python{VERSION}._pth" files. User can - move/rename *zip* file and edit `sys.path` by editing *_pth* file. - Here the `pattern` is used only for the default *zip* file name! - """ # noqa: D205 pattern = f"*python{interpreter.version_nodot}.zip" matches = fnmatch.filter(interpreter.path, pattern) matched_paths = map(Path, matches) existing_paths = filter(method("exists"), matched_paths) - path = next(existing_paths, None) - if path is not None: + if (path := next(existing_paths, None)) is not None: yield PathRefToDest(path, cls.to_bin) diff --git a/src/virtualenv/create/via_global_ref/builtin/cpython/mac_os.py b/src/virtualenv/create/via_global_ref/builtin/cpython/mac_os.py index 0ddbf9a33..c6beba8b5 100644 --- a/src/virtualenv/create/via_global_ref/builtin/cpython/mac_os.py +++ b/src/virtualenv/create/via_global_ref/builtin/cpython/mac_os.py @@ -9,6 +9,7 @@ from abc import ABC, abstractmethod from pathlib import Path from textwrap import dedent +from typing import TYPE_CHECKING from virtualenv.create.via_global_ref.builtin.ref import ( ExePathRefToDest, @@ -20,18 +21,25 @@ from .common import CPython, CPythonPosix, is_mac_os_framework, is_macos_brew from .cpython3 import CPython3 +if TYPE_CHECKING: + from collections.abc import Callable, Generator + from io import BufferedRandom + + from python_discovery import PythonInfo + + from virtualenv.create.via_global_ref.builtin.ref import PathRef + LOGGER = logging.getLogger(__name__) class CPythonmacOsFramework(CPython, ABC): @classmethod - def can_describe(cls, interpreter): + def can_describe(cls, interpreter: PythonInfo) -> bool: return is_mac_os_framework(interpreter) and super().can_describe(interpreter) - def create(self): + def create(self) -> None: super().create() - # change the install_name of the copied python executables target = self.desired_mach_o_image_path() current = self.current_mach_o_image_path() for src in self._sources: @@ -40,44 +48,48 @@ def create(self): if not self.symlinks: exes.extend(self.bin_dir / a for a in src.aliases) for exe in exes: - fix_mach_o(str(exe), current, target, self.interpreter.max_size) + fix_mach_o(str(exe), current, target, self.interpreter.max_size) # ty: ignore[invalid-argument-type] + try: + subprocess.check_call(["codesign", "--force", "--sign", "-", str(exe)]) # ruff:ignore[start-process-with-partial-path] + except (OSError, subprocess.CalledProcessError) as e: + LOGGER.warning("Could not ad-hoc re-sign %s: %s", exe, e) @classmethod - def _executables(cls, interpreter): + def _executables(cls, interpreter: PythonInfo) -> Generator[tuple[Path, list[str], str, str]]: for _, targets, must, when in super()._executables(interpreter): # Make sure we use the embedded interpreter inside the framework, even if sys.executable points to the # stub executable in ${sys.prefix}/bin. # See http://groups.google.com/group/python-virtualenv/browse_thread/thread/17cab2f85da75951 - fixed_host_exe = Path(interpreter.prefix) / "Resources" / "Python.app" / "Contents" / "MacOS" / "Python" + fixed_host_exe = Path(interpreter.prefix) / "Resources" / "Python.app" / "Contents" / "MacOS" / "Python" # ty: ignore[invalid-argument-type] yield fixed_host_exe, targets, must, when @abstractmethod - def current_mach_o_image_path(self): + def current_mach_o_image_path(self) -> str: raise NotImplementedError @abstractmethod - def desired_mach_o_image_path(self): + def desired_mach_o_image_path(self) -> str: raise NotImplementedError class CPython3macOsFramework(CPythonmacOsFramework, CPython3, CPythonPosix): - def current_mach_o_image_path(self): + def current_mach_o_image_path(self) -> str: return "@executable_path/../../../../Python3" - def desired_mach_o_image_path(self): + def desired_mach_o_image_path(self) -> str: return "@executable_path/../.Python" @classmethod - def sources(cls, interpreter): + def sources(cls, interpreter: PythonInfo) -> Generator[PathRef]: # ty: ignore[invalid-method-override] yield from super().sources(interpreter) # add a symlink to the host python image - exe = Path(interpreter.prefix) / "Python3" + exe = Path(interpreter.prefix) / "Python3" # ty: ignore[invalid-argument-type] yield PathRefToDest(exe, dest=lambda self, _: self.dest / ".Python", must=RefMust.SYMLINK) @property - def reload_code(self): - result = super().reload_code + def reload_code(self) -> str: + result = super().reload_code # ty: ignore[unresolved-attribute] return dedent( f""" # the bundled site.py always adds the global site package if we're on python framework build, escape this @@ -92,9 +104,8 @@ def reload_code(self): ) -def fix_mach_o(exe, current, new, max_size): - """ - https://en.wikipedia.org/wiki/Mach-O. +def fix_mach_o(exe: str, current: str, new: str, max_size: int) -> None: + """https://en.wikipedia.org/wiki/Mach-O. Mach-O, short for Mach object file format, is a file format for executables, object code, shared libraries, dynamically-loaded code, and core dumps. A replacement for the a.out format, Mach-O offers more extensibility and @@ -110,16 +121,17 @@ def fix_mach_o(exe, current, new, max_size): Lisp. With the introduction of Mac OS X 10.6 platform the Mach-O file underwent a significant modification that causes - binaries compiled on a computer running 10.6 or later to be (by default) executable only on computers running Mac - OS X 10.6 or later. The difference stems from load commands that the dynamic linker, in previous Mac OS X versions, + binaries compiled on a computer running 10.6 or later to be (by default) executable only on computers running Mac OS + X 10.6 or later. The difference stems from load commands that the dynamic linker, in previous Mac OS X versions, does not understand. Another significant change to the Mach-O format is the change in how the Link Edit tables (found in the __LINKEDIT section) function. In 10.6 these new Link Edit tables are compressed by removing unused and unneeded bits of information, however Mac OS X 10.5 and earlier cannot read this new Link Edit table format. + """ try: LOGGER.debug("change Mach-O for %s from %s to %s", exe, current, new) _builtin_change_mach_o(max_size)(exe, current, new) - except Exception as e: # noqa: BLE001 + except Exception as e: # ruff:ignore[blind-except] LOGGER.warning("Could not call _builtin_change_mac_o: %s. Trying to call install_name_tool instead.", e) try: cmd = ["install_name_tool", "-change", current, new, exe] @@ -129,22 +141,22 @@ def fix_mach_o(exe, current, new, max_size): raise -def _builtin_change_mach_o(maxint): # noqa: C901 - MH_MAGIC = 0xFEEDFACE # noqa: N806 - MH_CIGAM = 0xCEFAEDFE # noqa: N806 - MH_MAGIC_64 = 0xFEEDFACF # noqa: N806 - MH_CIGAM_64 = 0xCFFAEDFE # noqa: N806 - FAT_MAGIC = 0xCAFEBABE # noqa: N806 - BIG_ENDIAN = ">" # noqa: N806 - LITTLE_ENDIAN = "<" # noqa: N806 - LC_LOAD_DYLIB = 0xC # noqa: N806 +def _builtin_change_mach_o(maxint: int) -> Callable[[str, str, str], None]: # ruff:ignore[complex-structure] + MH_MAGIC = 0xFEEDFACE # ruff:ignore[non-lowercase-variable-in-function] + MH_CIGAM = 0xCEFAEDFE # ruff:ignore[non-lowercase-variable-in-function] + MH_MAGIC_64 = 0xFEEDFACF # ruff:ignore[non-lowercase-variable-in-function] + MH_CIGAM_64 = 0xCFFAEDFE # ruff:ignore[non-lowercase-variable-in-function] + FAT_MAGIC = 0xCAFEBABE # ruff:ignore[non-lowercase-variable-in-function] + BIG_ENDIAN = ">" # ruff:ignore[non-lowercase-variable-in-function] + LITTLE_ENDIAN = "<" # ruff:ignore[non-lowercase-variable-in-function] + LC_LOAD_DYLIB = 0xC # ruff:ignore[non-lowercase-variable-in-function] class FileView: """A proxy for file-like objects that exposes a given view of a file. Modified from macholib.""" - def __init__(self, file_obj, start=0, size=maxint) -> None: + def __init__(self, file_obj: FileView | BufferedRandom, start: int = 0, size: int = maxint) -> None: if isinstance(file_obj, FileView): - self._file_obj = file_obj._file_obj # noqa: SLF001 + self._file_obj = file_obj._file_obj # ruff:ignore[private-member-access] else: self._file_obj = file_obj self._start = start @@ -154,15 +166,15 @@ def __init__(self, file_obj, start=0, size=maxint) -> None: def __repr__(self) -> str: return f"" - def tell(self): + def tell(self) -> int: return self._pos - def _checkwindow(self, seek_to, op): + def _checkwindow(self, seek_to: int, op: str) -> None: if not (self._start <= seek_to <= self._end): msg = f"{op} to offset {seek_to:d} is outside window [{self._start:d}, {self._end:d}]" raise OSError(msg) - def seek(self, offset, whence=0): + def seek(self, offset: int, whence: int = 0) -> None: seek_to = offset if whence == os.SEEK_SET: seek_to += self._start @@ -177,7 +189,7 @@ def seek(self, offset, whence=0): self._file_obj.seek(seek_to) self._pos = seek_to - self._start - def write(self, content): + def write(self, content: bytes) -> None: here = self._start + self._pos self._checkwindow(here, "write") self._checkwindow(here + len(content), "write") @@ -185,8 +197,8 @@ def write(self, content): self._file_obj.write(content) self._pos += len(content) - def read(self, size=maxint): - assert size >= 0 # noqa: S101 + def read(self, size: int = maxint) -> bytes: + assert size >= 0 # ruff:ignore[assert] here = self._start + self._pos self._checkwindow(here, "read") size = min(size, self._end - here) @@ -195,55 +207,52 @@ def read(self, size=maxint): self._pos += len(read_bytes) return read_bytes - def read_data(file, endian, num=1): + def read_data(file: FileView, endian: str, num: int = 1) -> int | tuple[int, ...]: """Read a given number of 32-bits unsigned integers from the given file with the given endianness.""" res = struct.unpack(endian + "L" * num, file.read(num * 4)) if len(res) == 1: return res[0] return res - def mach_o_change(at_path, what, value): # noqa: C901 - """ - Replace a given name (what) in any LC_LOAD_DYLIB command found in the given binary with a new name (value), - provided it's shorter. - """ # noqa: D205 + def mach_o_change(at_path: str, what: str, value: str) -> None: # ruff:ignore[complex-structure] + """Replace a given name (what) in any LC_LOAD_DYLIB command found in the given binary with a new name (value), provided it's shorter.""" - def do_macho(file, bits, endian): + def do_macho(file: FileView, bits: int, endian: str) -> None: # Read Mach-O header (the magic number is assumed read by the caller) - _cpu_type, _cpu_sub_type, _file_type, n_commands, _size_of_commands, _flags = read_data(file, endian, 6) + _cpu_type, _cpu_sub_type, _file_type, n_commands, _size_of_commands, _flags = read_data(file, endian, 6) # ty: ignore[not-iterable] # 64-bits header has one more field. - if bits == 64: # noqa: PLR2004 + if bits == 64: # ruff:ignore[magic-value-comparison] read_data(file, endian) # The header is followed by n commands for _ in range(n_commands): where = file.tell() # Read command header - cmd, cmd_size = read_data(file, endian, 2) + cmd, cmd_size = read_data(file, endian, 2) # ty: ignore[not-iterable] if cmd == LC_LOAD_DYLIB: # The first data field in LC_LOAD_DYLIB commands is the offset of the name, starting from the # beginning of the command. name_offset = read_data(file, endian) - file.seek(where + name_offset, os.SEEK_SET) + file.seek(where + name_offset, os.SEEK_SET) # ty: ignore[unsupported-operator] # Read the NUL terminated string - load = file.read(cmd_size - name_offset).decode() + load = file.read(cmd_size - name_offset).decode() # ty: ignore[unsupported-operator] load = load[: load.index("\0")] # If the string is what is being replaced, overwrite it. if load == what: - file.seek(where + name_offset, os.SEEK_SET) + file.seek(where + name_offset, os.SEEK_SET) # ty: ignore[unsupported-operator] file.write(value.encode() + b"\0") # Seek to the next command file.seek(where + cmd_size, os.SEEK_SET) - def do_file(file, offset=0, size=maxint): + def do_file(file: FileView | BufferedRandom, offset: int = 0, size: int = maxint) -> None: file = FileView(file, offset, size) # Read magic number magic = read_data(file, BIG_ENDIAN) if magic == FAT_MAGIC: # Fat binaries contain nfat_arch Mach-O binaries n_fat_arch = read_data(file, BIG_ENDIAN) - for _ in range(n_fat_arch): + for _ in range(n_fat_arch): # ty: ignore[invalid-argument-type] # Read arch header - _cpu_type, _cpu_sub_type, offset, size, _align = read_data(file, BIG_ENDIAN, 5) + _cpu_type, _cpu_sub_type, offset, size, _align = read_data(file, BIG_ENDIAN, 5) # ty: ignore[not-iterable] do_file(file, offset, size) elif magic == MH_MAGIC: do_macho(file, 32, BIG_ENDIAN) @@ -254,7 +263,7 @@ def do_file(file, offset=0, size=maxint): elif magic == MH_CIGAM_64: do_macho(file, 64, LITTLE_ENDIAN) - assert len(what) >= len(value) # noqa: S101 + assert len(what) >= len(value) # ruff:ignore[assert] with open(at_path, "r+b") as f: do_file(f) @@ -264,11 +273,11 @@ def do_file(file, offset=0, size=maxint): class CPython3macOsBrew(CPython3, CPythonPosix): @classmethod - def can_describe(cls, interpreter): + def can_describe(cls, interpreter: PythonInfo) -> bool: return is_macos_brew(interpreter) and super().can_describe(interpreter) @classmethod - def setup_meta(cls, interpreter): # noqa: ARG003 + def setup_meta(cls, interpreter: PythonInfo) -> BuiltinViaGlobalRefMeta: # ruff:ignore[unused-class-method-argument] meta = BuiltinViaGlobalRefMeta() meta.copy_error = "Brew disables copy creation: https://github.com/Homebrew/homebrew-core/issues/138159" return meta diff --git a/src/virtualenv/create/via_global_ref/builtin/graalpy/__init__.py b/src/virtualenv/create/via_global_ref/builtin/graalpy/__init__.py index 8bfe887e7..9277c32a3 100644 --- a/src/virtualenv/create/via_global_ref/builtin/graalpy/__init__.py +++ b/src/virtualenv/create/via_global_ref/builtin/graalpy/__init__.py @@ -1,24 +1,36 @@ from __future__ import annotations -from abc import ABC +from abc import ABC, abstractmethod from pathlib import Path +from typing import TYPE_CHECKING from virtualenv.create.describe import PosixSupports, WindowsSupports from virtualenv.create.via_global_ref.builtin.ref import PathRefToDest, RefMust, RefWhen from virtualenv.create.via_global_ref.builtin.via_global_self_do import ViaGlobalRefVirtualenvBuiltin +if TYPE_CHECKING: + from collections.abc import Generator, Iterator + + from python_discovery import PythonInfo + class GraalPy(ViaGlobalRefVirtualenvBuiltin, ABC): @classmethod - def can_describe(cls, interpreter): - return interpreter.implementation == "GraalVM" and super().can_describe(interpreter) + @abstractmethod + def _native_lib(cls, lib_dir: Path, platform: str) -> Path: + """Return the path to the native library for this platform.""" + raise NotImplementedError + + @classmethod + def can_describe(cls, interpreter: PythonInfo) -> bool: + return interpreter.implementation == "GraalPy" and super().can_describe(interpreter) @classmethod - def exe_stem(cls): + def exe_stem(cls) -> str: return "graalpy" @classmethod - def exe_names(cls, interpreter): + def exe_names(cls, interpreter: PythonInfo) -> set[str]: return { cls.exe_stem(), "python", @@ -27,15 +39,15 @@ def exe_names(cls, interpreter): } @classmethod - def _executables(cls, interpreter): - host = Path(interpreter.system_executable) + def _executables(cls, interpreter: PythonInfo) -> Generator[tuple[Path, list[str], RefMust, RefWhen], None, None]: # ty: ignore[invalid-method-override] + host = Path(interpreter.system_exe) targets = sorted(f"{name}{cls.suffix}" for name in cls.exe_names(interpreter)) yield host, targets, RefMust.NA, RefWhen.ANY @classmethod - def sources(cls, interpreter): + def sources(cls, interpreter: PythonInfo) -> Generator[PathRefToDest]: # ty: ignore[invalid-method-override] yield from super().sources(interpreter) - python_dir = Path(interpreter.system_executable).resolve().parent + python_dir = Path(interpreter.system_exe).resolve().parent if python_dir.name in {"bin", "Scripts"}: python_dir = python_dir.parent @@ -49,14 +61,14 @@ def sources(cls, interpreter): yield PathRefToDest(jvm_dir, dest=lambda self, s: self.bin_dir.parent / s.name) @classmethod - def _shared_libs(cls, python_dir): + def _shared_libs(cls, python_dir: Path) -> Iterator[Path]: raise NotImplementedError - def set_pyenv_cfg(self): + def set_pyenv_cfg(self) -> None: super().set_pyenv_cfg() # GraalPy 24.0 and older had home without the bin version = self.interpreter.version_info - if version.major == 3 and version.minor <= 10: # noqa: PLR2004 + if version.minor <= 10: # ruff:ignore[magic-value-comparison] home = Path(self.pyenv_cfg["home"]) if home.name == "bin": self.pyenv_cfg["home"] = str(home.parent) @@ -64,7 +76,7 @@ def set_pyenv_cfg(self): class GraalPyPosix(GraalPy, PosixSupports): @classmethod - def _native_lib(cls, lib_dir, platform): + def _native_lib(cls, lib_dir: Path, platform: str) -> Path: if platform == "darwin": return lib_dir / "libpythonvm.dylib" return lib_dir / "libpythonvm.so" @@ -72,13 +84,13 @@ def _native_lib(cls, lib_dir, platform): class GraalPyWindows(GraalPy, WindowsSupports): @classmethod - def _native_lib(cls, lib_dir, _platform): + def _native_lib(cls, lib_dir: Path, platform: str) -> Path: # ruff:ignore[unused-class-method-argument] return lib_dir / "pythonvm.dll" - def set_pyenv_cfg(self): + def set_pyenv_cfg(self) -> None: # GraalPy needs an additional entry in pyvenv.cfg on Windows super().set_pyenv_cfg() - self.pyenv_cfg["venvlauncher_command"] = self.interpreter.system_executable + self.pyenv_cfg["venvlauncher_command"] = self.interpreter.system_exe __all__ = [ diff --git a/src/virtualenv/create/via_global_ref/builtin/pypy/common.py b/src/virtualenv/create/via_global_ref/builtin/pypy/common.py index ca4b45ff1..191f26267 100644 --- a/src/virtualenv/create/via_global_ref/builtin/pypy/common.py +++ b/src/virtualenv/create/via_global_ref/builtin/pypy/common.py @@ -2,28 +2,36 @@ import abc from pathlib import Path +from typing import TYPE_CHECKING from virtualenv.create.via_global_ref.builtin.ref import PathRefToDest, RefMust, RefWhen from virtualenv.create.via_global_ref.builtin.via_global_self_do import ViaGlobalRefVirtualenvBuiltin +if TYPE_CHECKING: + from collections.abc import Generator, Iterator + + from python_discovery import PythonInfo + + from virtualenv.create.via_global_ref.builtin.ref import PathRef + class PyPy(ViaGlobalRefVirtualenvBuiltin, abc.ABC): @classmethod - def can_describe(cls, interpreter): + def can_describe(cls, interpreter: PythonInfo) -> bool: return interpreter.implementation == "PyPy" and super().can_describe(interpreter) @classmethod - def _executables(cls, interpreter): - host = Path(interpreter.system_executable) + def _executables(cls, interpreter: PythonInfo) -> Generator[tuple[Path, list[str], str, str]]: + host = Path(interpreter.system_exe) targets = sorted(f"{name}{PyPy.suffix}" for name in cls.exe_names(interpreter)) yield host, targets, RefMust.NA, RefWhen.ANY @classmethod - def executables(cls, interpreter): + def executables(cls, interpreter: PythonInfo) -> Generator[PathRef]: yield from super().sources(interpreter) @classmethod - def exe_names(cls, interpreter): + def exe_names(cls, interpreter: PythonInfo) -> set[str]: return { cls.exe_stem(), "python", @@ -32,19 +40,19 @@ def exe_names(cls, interpreter): } @classmethod - def sources(cls, interpreter): + def sources(cls, interpreter: PythonInfo) -> Generator[PathRef]: # ty: ignore[invalid-method-override] yield from cls.executables(interpreter) for host in cls._add_shared_libs(interpreter): yield PathRefToDest(host, dest=lambda self, s: self.bin_dir / s.name) @classmethod - def _add_shared_libs(cls, interpreter): + def _add_shared_libs(cls, interpreter: PythonInfo) -> Generator[Path]: # https://bitbucket.org/pypy/pypy/issue/1922/future-proofing-virtualenv - python_dir = Path(interpreter.system_executable).resolve().parent + python_dir = Path(interpreter.system_exe).resolve().parent yield from cls._shared_libs(python_dir) @classmethod - def _shared_libs(cls, python_dir): + def _shared_libs(cls, python_dir: Path) -> Iterator[Path]: raise NotImplementedError diff --git a/src/virtualenv/create/via_global_ref/builtin/pypy/pypy3.py b/src/virtualenv/create/via_global_ref/builtin/pypy/pypy3.py index fa61ebc95..2587a7ae7 100644 --- a/src/virtualenv/create/via_global_ref/builtin/pypy/pypy3.py +++ b/src/virtualenv/create/via_global_ref/builtin/pypy/pypy3.py @@ -2,20 +2,28 @@ import abc from pathlib import Path +from typing import TYPE_CHECKING from virtualenv.create.describe import PosixSupports, Python3Supports, WindowsSupports from virtualenv.create.via_global_ref.builtin.ref import PathRefToDest from .common import PyPy +if TYPE_CHECKING: + from collections.abc import Generator, Iterator + + from python_discovery import PythonInfo + + from virtualenv.create.via_global_ref.builtin.ref import PathRef + class PyPy3(PyPy, Python3Supports, abc.ABC): @classmethod - def exe_stem(cls): + def exe_stem(cls) -> str: return "pypy3" @classmethod - def exe_names(cls, interpreter): + def exe_names(cls, interpreter: PythonInfo) -> set[str]: return super().exe_names(interpreter) | {"pypy"} @@ -23,50 +31,45 @@ class PyPy3Posix(PyPy3, PosixSupports): """PyPy 3 on POSIX.""" @classmethod - def _shared_libs(cls, python_dir): + def _shared_libs(cls, python_dir: Path) -> Iterator[Path]: # glob for libpypy3-c.so, libpypy3-c.dylib, libpypy3.9-c.so ... return python_dir.glob("libpypy3*.*") - def to_lib(self, src): + def to_lib(self, src: Path) -> Path: return self.dest / "lib" / src.name @classmethod - def sources(cls, interpreter): + def sources(cls, interpreter: PythonInfo) -> Generator[PathRef]: yield from super().sources(interpreter) - # PyPy >= 3.8 supports a standard prefix installation, where older - # versions always used a portable/development style installation. - # If this is a standard prefix installation, skip the below: + # PyPy >= 3.8 supports a standard prefix installation, where older versions always used a portable/development + # style installation. If this is a standard prefix installation, skip the below: if interpreter.system_prefix == "/usr": return - # Also copy/symlink anything under prefix/lib, which, for "portable" - # PyPy builds, includes the tk,tcl runtime and a number of shared - # objects. In distro-specific builds or on conda this should be empty - # (on PyPy3.8+ it will, like on CPython, hold the stdlib). + # Also copy/symlink anything under prefix/lib, which, for "portable" PyPy builds, includes the tk,tcl runtime + # and a number of shared objects. In distro-specific builds or on conda this should be empty (on PyPy3.8+ it + # will, like on CPython, hold the stdlib). host_lib = Path(interpreter.system_prefix) / "lib" stdlib = Path(interpreter.system_stdlib) if host_lib.exists() and host_lib.is_dir(): - for path in host_lib.iterdir(): - if stdlib == path: - # For PyPy3.8+ the stdlib lives in lib/pypy3.8 - # We need to avoid creating a symlink to it since that - # will defeat the purpose of a virtualenv - continue - yield PathRefToDest(path, dest=cls.to_lib) + if (deps_file := host_lib / "PYPY_PORTABLE_DEPS.txt").exists(): + for line in deps_file.read_text(encoding="utf-8").splitlines(): + dep = line.strip() + if dep and (path := host_lib / dep).exists(): + yield PathRefToDest(path, dest=cls.to_lib) + else: + for path in host_lib.iterdir(): + if stdlib == path: + continue + yield PathRefToDest(path, dest=cls.to_lib) class Pypy3Windows(PyPy3, WindowsSupports): """PyPy 3 on Windows.""" - @property - def less_v37(self): - return self.interpreter.version_info.minor < 7 # noqa: PLR2004 - @classmethod - def _shared_libs(cls, python_dir): - # glob for libpypy*.dll and libffi*.dll - for pattern in ["libpypy*.dll", "libffi*.dll"]: - srcs = python_dir.glob(pattern) - yield from srcs + def _shared_libs(cls, python_dir: Path) -> Iterator[Path]: + # PyPy does not use a PEP 397 launcher, so all DLLs from the interpreter directory are needed for the venv + yield from python_dir.glob("*.dll") __all__ = [ diff --git a/src/virtualenv/create/via_global_ref/builtin/ref.py b/src/virtualenv/create/via_global_ref/builtin/ref.py index e2fd45ffe..057c741cd 100644 --- a/src/virtualenv/create/via_global_ref/builtin/ref.py +++ b/src/virtualenv/create/via_global_ref/builtin/ref.py @@ -1,27 +1,37 @@ -""" -Virtual environments in the traditional sense are built as reference to the host python. This file allows declarative -references to elements on the file system, allowing our system to automatically detect what modes it can support given -the constraints: e.g. can the file system symlink, can the files be read, executed, etc. -""" # noqa: D205 +"""Virtual environments in the traditional sense are built as reference to the host python. This file allows declarative references to elements on the file system, allowing our system to automatically detect what modes it can support given the constraints: e.g. can the file system symlink, can the files be read, executed, etc.""" from __future__ import annotations import os +import sys from abc import ABC, abstractmethod from collections import OrderedDict from stat import S_IXGRP, S_IXOTH, S_IXUSR +from typing import TYPE_CHECKING from virtualenv.info import fs_is_case_sensitive, fs_supports_symlink from virtualenv.util.path import copy, make_exe, symlink +if TYPE_CHECKING: + from collections.abc import Callable + from pathlib import Path -class RefMust: +if sys.version_info >= (3, 11): # pragma: no cover (py311+) + from enum import StrEnum +else: # pragma: no cover (py311+) + from enum import Enum + + class StrEnum(str, Enum): + pass + + +class RefMust(StrEnum): NA = "NA" COPY = "copy" SYMLINK = "symlink" -class RefWhen: +class RefWhen(StrEnum): ANY = "ANY" COPY = "copy" SYMLINK = "symlink" @@ -33,7 +43,7 @@ class PathRef(ABC): FS_SUPPORTS_SYMLINK = fs_supports_symlink() FS_CASE_SENSITIVE = fs_is_case_sensitive() - def __init__(self, src, must=RefMust.NA, when=RefWhen.ANY) -> None: + def __init__(self, src: Path, must: str = RefMust.NA, when: str = RefWhen.ANY) -> None: self.must = must self.when = when self.src = src @@ -41,15 +51,15 @@ def __init__(self, src, must=RefMust.NA, when=RefWhen.ANY) -> None: self.exists = src.exists() except OSError: self.exists = False - self._can_read = None if self.exists else False - self._can_copy = None if self.exists else False - self._can_symlink = None if self.exists else False + self._can_read: bool | None = None if self.exists else False + self._can_copy: bool | None = None if self.exists else False + self._can_symlink: bool | None = None if self.exists else False def __repr__(self) -> str: return f"{self.__class__.__name__}(src={self.src})" @property - def can_read(self): + def can_read(self) -> bool: if self._can_read is None: if self.src.is_file(): try: @@ -62,7 +72,7 @@ def can_read(self): return self._can_read @property - def can_copy(self): + def can_copy(self) -> bool: if self._can_copy is None: if self.must == RefMust.SYMLINK: self._can_copy = self.can_symlink @@ -71,7 +81,7 @@ def can_copy(self): return self._can_copy @property - def can_symlink(self): + def can_symlink(self) -> bool: if self._can_symlink is None: if self.must == RefMust.COPY: self._can_symlink = self.can_copy @@ -80,10 +90,10 @@ def can_symlink(self): return self._can_symlink @abstractmethod - def run(self, creator, symlinks): + def run(self, creator: object, symlinks: bool) -> None: raise NotImplementedError - def method(self, symlinks): + def method(self, symlinks: bool) -> Callable[..., None]: if self.must == RefMust.SYMLINK: return symlink if self.must == RefMust.COPY: @@ -94,37 +104,32 @@ def method(self, symlinks): class ExePathRef(PathRef, ABC): """Base class that checks if a executable can be references via symlink/copy.""" - def __init__(self, src, must=RefMust.NA, when=RefWhen.ANY) -> None: + def __init__(self, src: Path, must: str = RefMust.NA, when: str = RefWhen.ANY) -> None: super().__init__(src, must, when) - self._can_run = None + self._can_run: bool | None = None @property - def can_symlink(self): + def can_symlink(self) -> bool: if self.FS_SUPPORTS_SYMLINK: return self.can_run return False @property - def can_run(self): + def can_run(self) -> bool: if self._can_run is None: mode = self.src.stat().st_mode - for key in [S_IXUSR, S_IXGRP, S_IXOTH]: - if mode & key: - self._can_run = True - break - else: - self._can_run = False + self._can_run = bool(mode & (S_IXUSR | S_IXGRP | S_IXOTH)) return self._can_run class PathRefToDest(PathRef): """Link a path on the file system.""" - def __init__(self, src, dest, must=RefMust.NA, when=RefWhen.ANY) -> None: + def __init__(self, src: Path, dest: Callable[..., Path], must: str = RefMust.NA, when: str = RefWhen.ANY) -> None: super().__init__(src, must, when) self.dest = dest - def run(self, creator, symlinks): + def run(self, creator: object, symlinks: bool) -> None: dest = self.dest(creator, self.src) method = self.method(symlinks) dest_iterable = dest if isinstance(dest, list) else (dest,) @@ -137,7 +142,9 @@ def run(self, creator, symlinks): class ExePathRefToDest(PathRefToDest, ExePathRef): """Link a exe path on the file system.""" - def __init__(self, src, targets, dest, must=RefMust.NA, when=RefWhen.ANY) -> None: + def __init__( + self, src: Path, targets: list[str], dest: Callable[..., Path], must: str = RefMust.NA, when: str = RefWhen.ANY + ) -> None: ExePathRef.__init__(self, src, must, when) PathRefToDest.__init__(self, src, dest, must, when) if not self.FS_CASE_SENSITIVE: @@ -146,7 +153,7 @@ def __init__(self, src, targets, dest, must=RefMust.NA, when=RefWhen.ANY) -> Non self.aliases = targets[1:] self.dest = dest - def run(self, creator, symlinks): + def run(self, creator: object, symlinks: bool) -> None: bin_dir = self.dest(creator, self.src).parent dest = bin_dir / self.base method = self.method(symlinks) @@ -155,7 +162,7 @@ def run(self, creator, symlinks): make_exe(dest) for extra in self.aliases: link_file = bin_dir / extra - if link_file.exists(): + if link_file.is_symlink() or link_file.exists(): link_file.unlink() if symlinks: link_file.symlink_to(self.base) diff --git a/src/virtualenv/create/via_global_ref/builtin/rustpython/__init__.py b/src/virtualenv/create/via_global_ref/builtin/rustpython/__init__.py new file mode 100644 index 000000000..e46574928 --- /dev/null +++ b/src/virtualenv/create/via_global_ref/builtin/rustpython/__init__.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +from abc import ABC +from pathlib import Path +from typing import TYPE_CHECKING + +from virtualenv.create.describe import PosixSupports, Python3Supports, WindowsSupports +from virtualenv.create.via_global_ref.builtin.ref import RefMust, RefWhen +from virtualenv.create.via_global_ref.builtin.via_global_self_do import ViaGlobalRefVirtualenvBuiltin + +if TYPE_CHECKING: + from collections.abc import Generator + + from python_discovery import PythonInfo + + +class RustPython(ViaGlobalRefVirtualenvBuiltin, Python3Supports, ABC): + @classmethod + def can_describe(cls, interpreter: PythonInfo) -> bool: + return interpreter.implementation == "RustPython" and super().can_describe(interpreter) + + @classmethod + def exe_stem(cls) -> str: + return "rustpython" + + @classmethod + def exe_names(cls, interpreter: PythonInfo) -> set[str]: + return { + cls.exe_stem(), + "python", + f"python{interpreter.version_info.major}", + f"python{interpreter.version_info.major}.{interpreter.version_info.minor}", + } + + @classmethod + def _executables(cls, interpreter: PythonInfo) -> Generator[tuple[Path, list[str], RefMust, RefWhen], None, None]: # ty: ignore[invalid-method-override] + host = Path(interpreter.system_exe) + targets = sorted(f"{name}{cls.suffix}" for name in cls.exe_names(interpreter)) + yield host, targets, RefMust.NA, RefWhen.ANY + + +class RustPythonPosix(RustPython, PosixSupports): + """RustPython on POSIX.""" + + +class RustPythonWindows(RustPython, WindowsSupports): + """RustPython on Windows.""" + + +__all__ = [ + "RustPythonPosix", + "RustPythonWindows", +] diff --git a/src/virtualenv/create/via_global_ref/builtin/via_global_self_do.py b/src/virtualenv/create/via_global_ref/builtin/via_global_self_do.py index 2f7f2f11a..cbb999de2 100644 --- a/src/virtualenv/create/via_global_ref/builtin/via_global_self_do.py +++ b/src/virtualenv/create/via_global_ref/builtin/via_global_self_do.py @@ -1,31 +1,43 @@ from __future__ import annotations from abc import ABC +from typing import TYPE_CHECKING from virtualenv.create.via_global_ref.api import ViaGlobalRefApi, ViaGlobalRefMeta from virtualenv.create.via_global_ref.builtin.ref import ( ExePathRefToDest, - RefMust, RefWhen, ) from virtualenv.util.path import ensure_dir from .builtin_way import VirtualenvBuiltin +if TYPE_CHECKING: + from collections.abc import Generator + from pathlib import Path + + from python_discovery import PythonInfo + + from virtualenv.config.cli.parser import VirtualEnvOptions + from virtualenv.create.via_global_ref.builtin.ref import PathRef + from virtualenv.create.via_global_ref.venv import Venv + class BuiltinViaGlobalRefMeta(ViaGlobalRefMeta): def __init__(self) -> None: super().__init__() - self.sources = [] + self.sources: list[PathRef] = [] class ViaGlobalRefVirtualenvBuiltin(ViaGlobalRefApi, VirtualenvBuiltin, ABC): - def __init__(self, options, interpreter) -> None: + def __init__(self, options: VirtualEnvOptions, interpreter: PythonInfo) -> None: super().__init__(options, interpreter) - self._sources = getattr(options.meta, "sources", None) # if we're created as a describer this might be missing + self._sources: list[PathRef] = ( + getattr(options.meta, "sources", None) or [] + ) # if created as a describer this might be missing @classmethod - def can_create(cls, interpreter): + def can_create(cls, interpreter: PythonInfo) -> BuiltinViaGlobalRefMeta | None: """By default, all built-in methods assume that if we can describe it we can create it.""" # first we must be able to describe it if not cls.can_describe(interpreter): @@ -36,7 +48,7 @@ def can_create(cls, interpreter): return meta @classmethod - def _sources_can_be_applied(cls, interpreter, meta): + def _sources_can_be_applied(cls, interpreter: PythonInfo, meta: BuiltinViaGlobalRefMeta) -> None: for src in cls.sources(interpreter): if src.exists: if meta.can_copy and not src.can_copy: @@ -45,11 +57,11 @@ def _sources_can_be_applied(cls, interpreter, meta): meta.symlink_error = f"cannot symlink {src}" else: msg = f"missing required file {src}" - if src.when == RefMust.NA: + if src.when == RefWhen.ANY: meta.error = msg - elif src.when == RefMust.COPY: + elif src.when == RefWhen.COPY: meta.copy_error = msg - elif src.when == RefMust.SYMLINK: + elif src.when == RefWhen.SYMLINK: meta.symlink_error = msg if not meta.can_copy and not meta.can_symlink: meta.error = f"neither copy or symlink supported, copy: {meta.copy_error} symlink: {meta.symlink_error}" @@ -58,22 +70,22 @@ def _sources_can_be_applied(cls, interpreter, meta): meta.sources.append(src) @classmethod - def setup_meta(cls, interpreter): # noqa: ARG003 + def setup_meta(cls, interpreter: PythonInfo) -> BuiltinViaGlobalRefMeta: # ruff:ignore[unused-class-method-argument] return BuiltinViaGlobalRefMeta() @classmethod - def sources(cls, interpreter): + def sources(cls, interpreter: PythonInfo) -> Generator[ExePathRefToDest]: for host_exe, targets, must, when in cls._executables(interpreter): yield ExePathRefToDest(host_exe, dest=cls.to_bin, targets=targets, must=must, when=when) - def to_bin(self, src): + def to_bin(self, src: Path) -> Path: return self.bin_dir / src.name @classmethod - def _executables(cls, interpreter): + def _executables(cls, interpreter: PythonInfo) -> Generator[tuple[Path, list[str], str, str]]: raise NotImplementedError - def create(self): + def create(self) -> None: dirs = self.ensure_directories() for directory in list(dirs): if any(i for i in dirs if i is not directory and directory.parts == i.parts[: len(directory.parts)]): @@ -98,18 +110,22 @@ def create(self): self.enable_system_site_package = true_system_site super().create() - def ensure_directories(self): - return {self.dest, self.bin_dir, self.script_dir, self.stdlib} | set(self.libs) + @property + def include_dir(self) -> Path: + return self.dest / ("Include" if self.interpreter.os == "nt" else "include") + + def install_venv_shared_libs(self, venv_creator: Venv) -> None: + pass + + def ensure_directories(self) -> set[Path]: + return {self.dest, self.bin_dir, self.script_dir, self.stdlib, self.include_dir} | set(self.libs) - def set_pyenv_cfg(self): - """ - We directly inject the base prefix and base exec prefix to avoid site.py needing to discover these - from home (which usually is done within the interpreter itself). - """ # noqa: D205 + def set_pyenv_cfg(self) -> None: + """We directly inject the base prefix and base exec prefix to avoid site.py needing to discover these from home (which usually is done within the interpreter itself).""" super().set_pyenv_cfg() self.pyenv_cfg["base-prefix"] = self.interpreter.system_prefix self.pyenv_cfg["base-exec-prefix"] = self.interpreter.system_exec_prefix - self.pyenv_cfg["base-executable"] = self.interpreter.system_executable + self.pyenv_cfg["base-executable"] = self.interpreter.system_exe __all__ = [ diff --git a/src/virtualenv/create/via_global_ref/store.py b/src/virtualenv/create/via_global_ref/store.py index 4be668921..3c8280ed8 100644 --- a/src/virtualenv/create/via_global_ref/store.py +++ b/src/virtualenv/create/via_global_ref/store.py @@ -1,18 +1,24 @@ from __future__ import annotations from pathlib import Path +from typing import TYPE_CHECKING +if TYPE_CHECKING: + from python_discovery import PythonInfo -def handle_store_python(meta, interpreter): + from virtualenv.create.via_global_ref.api import ViaGlobalRefMeta + + +def handle_store_python(meta: ViaGlobalRefMeta, interpreter: PythonInfo) -> ViaGlobalRefMeta: if is_store_python(interpreter): meta.symlink_error = "Windows Store Python does not support virtual environments via symlink" return meta -def is_store_python(interpreter): - parts = Path(interpreter.system_executable).parts +def is_store_python(interpreter: PythonInfo) -> bool: + parts = Path(interpreter.system_exe).parts return ( - len(parts) > 4 # noqa: PLR2004 + len(parts) > 4 # ruff:ignore[magic-value-comparison] and parts[-4] == "Microsoft" and parts[-3] == "WindowsApps" and parts[-2].startswith("PythonSoftwareFoundation.Python.3.") diff --git a/src/virtualenv/create/via_global_ref/venv.py b/src/virtualenv/create/via_global_ref/venv.py index d5037bc3b..7a49d0469 100644 --- a/src/virtualenv/create/via_global_ref/venv.py +++ b/src/virtualenv/create/via_global_ref/venv.py @@ -2,65 +2,64 @@ import logging from copy import copy +from typing import TYPE_CHECKING + +from python_discovery import PythonInfo from virtualenv.create.via_global_ref.store import handle_store_python -from virtualenv.discovery.py_info import PythonInfo from virtualenv.util.error import ProcessCallFailedError from virtualenv.util.path import ensure_dir from virtualenv.util.subprocess import run_cmd from .api import ViaGlobalRefApi, ViaGlobalRefMeta +from .builtin.cpython.common import is_mac_os_framework from .builtin.cpython.mac_os import CPython3macOsBrew -from .builtin.pypy.pypy3 import Pypy3Windows + +if TYPE_CHECKING: + from typing import Any + + from virtualenv.config.cli.parser import VirtualEnvOptions LOGGER = logging.getLogger(__name__) class Venv(ViaGlobalRefApi): - def __init__(self, options, interpreter) -> None: + def __init__(self, options: VirtualEnvOptions, interpreter: PythonInfo) -> None: self.describe = options.describe super().__init__(options, interpreter) current = PythonInfo.current() self.can_be_inline = interpreter is current and interpreter.executable == interpreter.system_executable self._context = None - def _args(self): + def _args(self) -> list[tuple[str, Any]]: return super()._args() + ([("describe", self.describe.__class__.__name__)] if self.describe else []) @classmethod - def can_create(cls, interpreter): + def can_create(cls, interpreter: PythonInfo) -> ViaGlobalRefMeta | None: if interpreter.has_venv: if CPython3macOsBrew.can_describe(interpreter): return CPython3macOsBrew.setup_meta(interpreter) meta = ViaGlobalRefMeta() if interpreter.platform == "win32": meta = handle_store_python(meta, interpreter) + if is_mac_os_framework(interpreter): + meta.copy_error = "macOS framework builds do not support copy-based virtual environments" return meta return None - def create(self): + def create(self) -> None: if self.can_be_inline: self.create_inline() else: self.create_via_sub_process() - for lib in self.libs: + for lib in self.libs: # ty: ignore[not-iterable] ensure_dir(lib) + if self.describe is not None: + self.describe.install_venv_shared_libs(self) super().create() - self.executables_for_win_pypy_less_v37() - - def executables_for_win_pypy_less_v37(self): - """ - PyPy <= 3.6 (v7.3.3) for Windows contains only pypy3.exe and pypy3w.exe - Venv does not handle non-existing exe sources, e.g. python.exe, so this - patch does it. - """ # noqa: D205 - creator = self.describe - if isinstance(creator, Pypy3Windows) and creator.less_v37: - for exe in creator.executables(self.interpreter): - exe.run(creator, self.symlinks) - - def create_inline(self): - from venv import EnvBuilder # noqa: PLC0415 + + def create_inline(self) -> None: + from venv import EnvBuilder # ruff:ignore[import-outside-top-level] builder = EnvBuilder( system_site_packages=self.enable_system_site_package, @@ -70,27 +69,29 @@ def create_inline(self): ) builder.create(str(self.dest)) - def create_via_sub_process(self): + def create_via_sub_process(self) -> None: cmd = self.get_host_create_cmd() LOGGER.info("using host built-in venv to create via %s", " ".join(cmd)) code, out, err = run_cmd(cmd) if code != 0: raise ProcessCallFailedError(code, out, err, cmd) - def get_host_create_cmd(self): + def get_host_create_cmd(self) -> list[str]: cmd = [self.interpreter.system_executable, "-m", "venv", "--without-pip"] + if self.interpreter.version_info >= (3, 13): + cmd.append("--without-scm-ignore-files") if self.enable_system_site_package: cmd.append("--system-site-packages") cmd.extend(("--symlinks" if self.symlinks else "--copies", str(self.dest))) - return cmd + return cmd # ty: ignore[invalid-return-type] - def set_pyenv_cfg(self): + def set_pyenv_cfg(self) -> None: # prefer venv options over ours, but keep our extra venv_content = copy(self.pyenv_cfg.refresh()) super().set_pyenv_cfg() self.pyenv_cfg.update(venv_content) - def __getattribute__(self, item): + def __getattribute__(self, item: str) -> object: describe = object.__getattribute__(self, "describe") if describe is not None and hasattr(describe, item): element = getattr(describe, item) diff --git a/src/virtualenv/discovery/builtin.py b/src/virtualenv/discovery/builtin.py index e2d193911..395e77116 100644 --- a/src/virtualenv/discovery/builtin.py +++ b/src/virtualenv/discovery/builtin.py @@ -1,38 +1,43 @@ +"""Virtualenv-specific Builtin discovery wrapping py_discovery.""" + from __future__ import annotations -import logging -import os import sys -from contextlib import suppress -from pathlib import Path from typing import TYPE_CHECKING -from platformdirs import user_data_path - -from virtualenv.info import IS_WIN, fs_path_id +from python_discovery import get_interpreter as _get_interpreter from .discover import Discover -from .py_info import PythonInfo -from .py_spec import PythonSpec if TYPE_CHECKING: from argparse import ArgumentParser - from collections.abc import Callable, Generator, Iterable, Mapping, Sequence + from collections.abc import Iterable, Mapping, Sequence + + from python_discovery import PyInfoCache, PythonInfo + + from virtualenv.config.cli.parser import VirtualEnvOptions + - from virtualenv.app_data.base import AppData -LOGGER = logging.getLogger(__name__) +def get_interpreter( + key: str, + try_first_with: Iterable[str], + cache: PyInfoCache | None = None, + env: Mapping[str, str] | None = None, + app_data: PyInfoCache | None = None, +) -> PythonInfo | None: + return _get_interpreter(key, try_first_with, cache or app_data, env) class Builtin(Discover): python_spec: Sequence[str] - app_data: AppData + app_data: PyInfoCache try_first_with: Sequence[str] - def __init__(self, options) -> None: + def __init__(self, options: VirtualEnvOptions) -> None: super().__init__(options) self.python_spec = options.python or [sys.executable] if self._env.get("VIRTUALENV_PYTHON"): - self.python_spec = self.python_spec[1:] + self.python_spec[:1] # Rotate the list + self.python_spec = self.python_spec[1:] + self.python_spec[:1] self.app_data = options.app_data self.try_first_with = options.try_first_with @@ -46,8 +51,9 @@ def add_parser_arguments(cls, parser: ArgumentParser) -> None: type=str, action="append", default=[], - help="interpreter based on what to create environment (path/identifier) " - "- by default use the interpreter where the tool is installed - first found wins", + help="interpreter based on what to create environment (path/identifier/version-specifier) " + "- by default use the interpreter where the tool is installed - first found wins. " + "Version specifiers (e.g., >=3.12, ~=3.11.0, ==3.10) are also supported", ) parser.add_argument( "--try-first-with", @@ -61,8 +67,12 @@ def add_parser_arguments(cls, parser: ArgumentParser) -> None: def run(self) -> PythonInfo | None: for python_spec in self.python_spec: - result = get_interpreter(python_spec, self.try_first_with, self.app_data, self._env) - if result is not None: + if result := get_interpreter( + python_spec, + self.try_first_with, + app_data=self.app_data, + env=self._env, + ): return result return None @@ -71,195 +81,7 @@ def __repr__(self) -> str: return f"{self.__class__.__name__} discover of python_spec={spec!r}" -def get_interpreter( - key, try_first_with: Iterable[str], app_data: AppData | None = None, env: Mapping[str, str] | None = None -) -> PythonInfo | None: - spec = PythonSpec.from_string_spec(key) - LOGGER.info("find interpreter for spec %r", spec) - proposed_paths = set() - env = os.environ if env is None else env - for interpreter, impl_must_match in propose_interpreters(spec, try_first_with, app_data, env): - key = interpreter.system_executable, impl_must_match - if key in proposed_paths: - continue - LOGGER.info("proposed %s", interpreter) - if interpreter.satisfies(spec, impl_must_match): - LOGGER.debug("accepted %s", interpreter) - return interpreter - proposed_paths.add(key) - return None - - -def propose_interpreters( # noqa: C901, PLR0912, PLR0915 - spec: PythonSpec, - try_first_with: Iterable[str], - app_data: AppData | None = None, - env: Mapping[str, str] | None = None, -) -> Generator[tuple[PythonInfo, bool], None, None]: - # 0. if it's a path and exists, and is absolute path, this is the only option we consider - env = os.environ if env is None else env - tested_exes: set[str] = set() - if spec.is_abs: - try: - os.lstat(spec.path) # Windows Store Python does not work with os.path.exists, but does for os.lstat - except OSError: - pass - else: - exe_raw = os.path.abspath(spec.path) - exe_id = fs_path_id(exe_raw) - if exe_id not in tested_exes: - tested_exes.add(exe_id) - yield PythonInfo.from_exe(exe_raw, app_data, env=env), True - return - - # 1. try with first - for py_exe in try_first_with: - path = os.path.abspath(py_exe) - try: - os.lstat(path) # Windows Store Python does not work with os.path.exists, but does for os.lstat - except OSError: - pass - else: - exe_raw = os.path.abspath(path) - exe_id = fs_path_id(exe_raw) - if exe_id in tested_exes: - continue - tested_exes.add(exe_id) - yield PythonInfo.from_exe(exe_raw, app_data, env=env), True - - # 1. if it's a path and exists - if spec.path is not None: - try: - os.lstat(spec.path) # Windows Store Python does not work with os.path.exists, but does for os.lstat - except OSError: - pass - else: - exe_raw = os.path.abspath(spec.path) - exe_id = fs_path_id(exe_raw) - if exe_id not in tested_exes: - tested_exes.add(exe_id) - yield PythonInfo.from_exe(exe_raw, app_data, env=env), True - if spec.is_abs: - return - else: - # 2. otherwise try with the current - current_python = PythonInfo.current_system(app_data) - exe_raw = str(current_python.executable) - exe_id = fs_path_id(exe_raw) - if exe_id not in tested_exes: - tested_exes.add(exe_id) - yield current_python, True - - # 3. otherwise fallback to platform default logic - if IS_WIN: - from .windows import propose_interpreters # noqa: PLC0415 - - for interpreter in propose_interpreters(spec, app_data, env): - exe_raw = str(interpreter.executable) - exe_id = fs_path_id(exe_raw) - if exe_id in tested_exes: - continue - tested_exes.add(exe_id) - yield interpreter, True - - # try to find on path, the path order matters (as the candidates are less easy to control by end user) - find_candidates = path_exe_finder(spec) - for pos, path in enumerate(get_paths(env)): - LOGGER.debug(LazyPathDump(pos, path, env)) - for exe, impl_must_match in find_candidates(path): - exe_raw = str(exe) - exe_id = fs_path_id(exe_raw) - if exe_id in tested_exes: - continue - tested_exes.add(exe_id) - interpreter = PathPythonInfo.from_exe(exe_raw, app_data, raise_on_error=False, env=env) - if interpreter is not None: - yield interpreter, impl_must_match - - # otherwise try uv-managed python (~/.local/share/uv/python or platform equivalent) - if uv_python_dir := os.getenv("UV_PYTHON_INSTALL_DIR"): - uv_python_path = Path(uv_python_dir).expanduser() - elif xdg_data_home := os.getenv("XDG_DATA_HOME"): - uv_python_path = Path(xdg_data_home).expanduser() / "uv" / "python" - else: - uv_python_path = user_data_path("uv") / "python" - - for exe_path in uv_python_path.glob("*/bin/python"): - interpreter = PathPythonInfo.from_exe(str(exe_path), app_data, raise_on_error=False, env=env) - if interpreter is not None: - yield interpreter, True - - -def get_paths(env: Mapping[str, str]) -> Generator[Path, None, None]: - path = env.get("PATH", None) - if path is None: - try: - path = os.confstr("CS_PATH") - except (AttributeError, ValueError): - path = os.defpath - if path: - for p in map(Path, path.split(os.pathsep)): - with suppress(OSError): - if p.is_dir() and next(p.iterdir(), None): - yield p - - -class LazyPathDump: - def __init__(self, pos: int, path: Path, env: Mapping[str, str]) -> None: - self.pos = pos - self.path = path - self.env = env - - def __repr__(self) -> str: - content = f"discover PATH[{self.pos}]={self.path}" - if self.env.get("_VIRTUALENV_DEBUG"): # this is the over the board debug - content += " with =>" - for file_path in self.path.iterdir(): - try: - if file_path.is_dir(): - continue - if IS_WIN: - pathext = self.env.get("PATHEXT", ".COM;.EXE;.BAT;.CMD").split(";") - if not any(file_path.name.upper().endswith(ext) for ext in pathext): - continue - elif not (file_path.stat().st_mode & os.X_OK): - continue - except OSError: - pass - content += " " - content += file_path.name - return content - - -def path_exe_finder(spec: PythonSpec) -> Callable[[Path], Generator[tuple[Path, bool], None, None]]: - """Given a spec, return a function that can be called on a path to find all matching files in it.""" - pat = spec.generate_re(windows=sys.platform == "win32") - direct = spec.str_spec - if sys.platform == "win32": - direct = f"{direct}.exe" - - def path_exes(path: Path) -> Generator[tuple[Path, bool], None, None]: - # 4. then maybe it's something exact on PATH - if it was direct lookup implementation no longer counts - direct_path = path / direct - if direct_path.exists(): - yield direct_path, False - - # 5. or from the spec we can deduce if a name on path matches - for exe in path.iterdir(): - match = pat.fullmatch(exe.name) - if match: - # the implementation must match when we find “python[ver]” - yield exe.absolute(), match["impl"] == "python" - - return path_exes - - -class PathPythonInfo(PythonInfo): - """python info from path.""" - - __all__ = [ "Builtin", - "PathPythonInfo", "get_interpreter", ] diff --git a/src/virtualenv/discovery/cached_py_info.py b/src/virtualenv/discovery/cached_py_info.py index ee2034615..20873f3f9 100644 --- a/src/virtualenv/discovery/cached_py_info.py +++ b/src/virtualenv/discovery/cached_py_info.py @@ -1,194 +1,10 @@ -""" - -We acquire the python information by running an interrogation script via subprocess trigger. This operation is not -cheap, especially not on Windows. To not have to pay this hefty cost every time we apply multiple levels of -caching. -""" # noqa: D205 +"""Backward-compatibility re-export — use ``python_discovery`` directly.""" from __future__ import annotations -import hashlib -import logging -import os -import random -import sys -from collections import OrderedDict -from pathlib import Path -from shlex import quote -from string import ascii_lowercase, ascii_uppercase, digits -from subprocess import Popen - -from virtualenv.app_data import AppDataDisabled -from virtualenv.discovery.py_info import PythonInfo -from virtualenv.util.subprocess import subprocess - -_CACHE = OrderedDict() -_CACHE[Path(sys.executable)] = PythonInfo() -LOGGER = logging.getLogger(__name__) - - -def from_exe(cls, app_data, exe, env=None, raise_on_error=True, ignore_cache=False): # noqa: FBT002, PLR0913 - env = os.environ if env is None else env - result = _get_from_cache(cls, app_data, exe, env, ignore_cache=ignore_cache) - if isinstance(result, Exception): - if raise_on_error: - raise result - LOGGER.info("%s", result) - result = None - return result - - -def _get_from_cache(cls, app_data, exe, env, ignore_cache=True): # noqa: FBT002 - # note here we cannot resolve symlinks, as the symlink may trigger different prefix information if there's a - # pyenv.cfg somewhere alongside on python3.5+ - exe_path = Path(exe) - if not ignore_cache and exe_path in _CACHE: # check in the in-memory cache - result = _CACHE[exe_path] - else: # otherwise go through the app data cache - py_info = _get_via_file_cache(cls, app_data, exe_path, exe, env) - result = _CACHE[exe_path] = py_info - # independent if it was from the file or in-memory cache fix the original executable location - if isinstance(result, PythonInfo): - result.executable = exe - return result - - -def _get_via_file_cache(cls, app_data, path, exe, env): - path_text = str(path) - try: - path_modified = path.stat().st_mtime - except OSError: - path_modified = -1 - py_info_script = Path(os.path.abspath(__file__)).parent / "py_info.py" - try: - py_info_hash = hashlib.sha256(py_info_script.read_bytes()).hexdigest() - except OSError: - py_info_hash = None - - if app_data is None: - app_data = AppDataDisabled() - py_info, py_info_store = None, app_data.py_info(path) - with py_info_store.locked(): - if py_info_store.exists(): # if exists and matches load - data = py_info_store.read() - of_path = data.get("path") - of_st_mtime = data.get("st_mtime") - of_content = data.get("content") - of_hash = data.get("hash") - if of_path == path_text and of_st_mtime == path_modified and of_hash == py_info_hash: - py_info = cls._from_dict(of_content.copy()) - sys_exe = py_info.system_executable - if sys_exe is not None and not os.path.exists(sys_exe): - py_info_store.remove() - py_info = None - else: - py_info_store.remove() - if py_info is None: # if not loaded run and save - failure, py_info = _run_subprocess(cls, exe, app_data, env) - if failure is None: - data = { - "st_mtime": path_modified, - "path": path_text, - "content": py_info._to_dict(), # noqa: SLF001 - "hash": py_info_hash, - } - py_info_store.write(data) - else: - py_info = failure - return py_info - - -COOKIE_LENGTH: int = 32 - - -def gen_cookie(): - return "".join( - random.choice(f"{ascii_lowercase}{ascii_uppercase}{digits}") # noqa: S311 - for _ in range(COOKIE_LENGTH) - ) - - -def _run_subprocess(cls, exe, app_data, env): - py_info_script = Path(os.path.abspath(__file__)).parent / "py_info.py" - # Cookies allow to split the serialized stdout output generated by the script collecting the info from the output - # generated by something else. The right way to deal with it is to create an anonymous pipe and pass its descriptor - # to the child and output to it. But AFAIK all of them are either not cross-platform or too big to implement and are - # not in the stdlib. So the easiest and the shortest way I could mind is just using the cookies. - # We generate pseudorandom cookies because it easy to implement and avoids breakage from outputting modules source - # code, i.e. by debug output libraries. We reverse the cookies to avoid breakages resulting from variable values - # appearing in debug output. - - start_cookie = gen_cookie() - end_cookie = gen_cookie() - with app_data.ensure_extracted(py_info_script) as py_info_script: - cmd = [exe, str(py_info_script), start_cookie, end_cookie] - # prevent sys.prefix from leaking into the child process - see https://bugs.python.org/issue22490 - env = env.copy() - env.pop("__PYVENV_LAUNCHER__", None) - LOGGER.debug("get interpreter info via cmd: %s", LogCmd(cmd)) - try: - process = Popen( - cmd, - universal_newlines=True, - stdin=subprocess.PIPE, - stderr=subprocess.PIPE, - stdout=subprocess.PIPE, - env=env, - encoding="utf-8", - ) - out, err = process.communicate() - code = process.returncode - except OSError as os_error: - out, err, code = "", os_error.strerror, os_error.errno - result, failure = None, None - if code == 0: - out_starts = out.find(start_cookie[::-1]) +from python_discovery._cached_py_info import clear, from_exe # ruff:ignore[import-private-name] - if out_starts > -1: - pre_cookie = out[:out_starts] - - if pre_cookie: - sys.stdout.write(pre_cookie) - - out = out[out_starts + COOKIE_LENGTH :] - - out_ends = out.find(end_cookie[::-1]) - - if out_ends > -1: - post_cookie = out[out_ends + COOKIE_LENGTH :] - - if post_cookie: - sys.stdout.write(post_cookie) - - out = out[:out_ends] - - result = cls._from_json(out) - result.executable = exe # keep original executable as this may contain initialization code - else: - msg = f"{exe} with code {code}{f' out: {out!r}' if out else ''}{f' err: {err!r}' if err else ''}" - failure = RuntimeError(f"failed to query {msg}") - return failure, result - - -class LogCmd: - def __init__(self, cmd, env=None) -> None: - self.cmd = cmd - self.env = env - - def __repr__(self) -> str: - cmd_repr = " ".join(quote(str(c)) for c in self.cmd) - if self.env is not None: - cmd_repr = f"{cmd_repr} env of {self.env!r}" - return cmd_repr - - -def clear(app_data): - app_data.py_info_clear() - _CACHE.clear() - - -___all___ = [ - "from_exe", +__all__ = [ "clear", - "LogCmd", + "from_exe", ] diff --git a/src/virtualenv/discovery/discover.py b/src/virtualenv/discovery/discover.py index 0aaa17c8e..457c15903 100644 --- a/src/virtualenv/discovery/discover.py +++ b/src/virtualenv/discovery/discover.py @@ -1,42 +1,37 @@ +"""Virtualenv-specific Discover base class for plugin-based Python discovery.""" + from __future__ import annotations +import os from abc import ABC, abstractmethod +from typing import TYPE_CHECKING +if TYPE_CHECKING: + from argparse import ArgumentParser + from collections.abc import Mapping -class Discover(ABC): - """Discover and provide the requested Python interpreter.""" + from python_discovery import PythonInfo - @classmethod - def add_parser_arguments(cls, parser): - """ - Add CLI arguments for this discovery mechanisms. + from virtualenv.config.cli.parser import VirtualEnvOptions - :param parser: the CLI parser - """ - raise NotImplementedError - def __init__(self, options) -> None: - """ - Create a new discovery mechanism. +class Discover(ABC): + @classmethod + def add_parser_arguments(cls, parser: ArgumentParser) -> None: + raise NotImplementedError - :param options: the parsed options as defined within :meth:`add_parser_arguments` - """ + def __init__(self, options: VirtualEnvOptions) -> None: self._has_run = False - self._interpreter = None - self._env = options.env + self._interpreter: PythonInfo | None = None + self._env: Mapping[str, str] = options.env if options.env is not None else os.environ @abstractmethod - def run(self): - """ - Discovers an interpreter. - - :return: the interpreter ready to use for virtual environment creation - """ + def run(self) -> PythonInfo | None: raise NotImplementedError @property - def interpreter(self): - """:return: the interpreter as returned by :meth:`run`, cached""" + def interpreter(self) -> PythonInfo | None: + """The interpreter as returned by :meth:`run`, cached.""" if self._has_run is False: self._interpreter = self.run() self._has_run = True diff --git a/src/virtualenv/discovery/py_info.py b/src/virtualenv/discovery/py_info.py index c2310cd7e..511586593 100644 --- a/src/virtualenv/discovery/py_info.py +++ b/src/virtualenv/discovery/py_info.py @@ -1,673 +1,9 @@ -""" -The PythonInfo contains information about a concrete instance of a Python interpreter. - -Note: this file is also used to query target interpreters, so can only use standard library methods -""" +"""Backward-compatibility re-export — use ``python_discovery.PythonInfo`` directly.""" from __future__ import annotations -import json -import logging -import os -import platform -import re -import struct -import sys -import sysconfig -import warnings -from collections import OrderedDict, namedtuple -from string import digits - -VersionInfo = namedtuple("VersionInfo", ["major", "minor", "micro", "releaselevel", "serial"]) # noqa: PYI024 -LOGGER = logging.getLogger(__name__) - - -def _get_path_extensions(): - return list(OrderedDict.fromkeys(["", *os.environ.get("PATHEXT", "").lower().split(os.pathsep)])) - - -EXTENSIONS = _get_path_extensions() -_CONF_VAR_RE = re.compile(r"\{\w+}") - - -class PythonInfo: # noqa: PLR0904 - """Contains information for a Python interpreter.""" - - def __init__(self) -> None: # noqa: PLR0915 - def abs_path(v): - return None if v is None else os.path.abspath(v) # unroll relative elements from path (e.g. ..) - - # qualifies the python - self.platform = sys.platform - self.implementation = platform.python_implementation() - if self.implementation == "PyPy": - self.pypy_version_info = tuple(sys.pypy_version_info) - - # this is a tuple in earlier, struct later, unify to our own named tuple - self.version_info = VersionInfo(*sys.version_info) - # Use the same implementation as found in stdlib platform.architecture - # to account for platforms where the maximum integer is not equal the - # pointer size. - self.architecture = 32 if struct.calcsize("P") == 4 else 64 # noqa: PLR2004 - - # Used to determine some file names. - # See `CPython3Windows.python_zip()`. - self.version_nodot = sysconfig.get_config_var("py_version_nodot") - - self.version = sys.version - self.os = os.name - self.free_threaded = sysconfig.get_config_var("Py_GIL_DISABLED") == 1 - - # information about the prefix - determines python home - self.prefix = abs_path(getattr(sys, "prefix", None)) # prefix we think - self.base_prefix = abs_path(getattr(sys, "base_prefix", None)) # venv - self.real_prefix = abs_path(getattr(sys, "real_prefix", None)) # old virtualenv - - # information about the exec prefix - dynamic stdlib modules - self.base_exec_prefix = abs_path(getattr(sys, "base_exec_prefix", None)) - self.exec_prefix = abs_path(getattr(sys, "exec_prefix", None)) - - self.executable = abs_path(sys.executable) # the executable we were invoked via - self.original_executable = abs_path(self.executable) # the executable as known by the interpreter - self.system_executable = self._fast_get_system_executable() # the executable we are based of (if available) - - try: - __import__("venv") - has = True - except ImportError: - has = False - self.has_venv = has - self.path = sys.path - self.file_system_encoding = sys.getfilesystemencoding() - self.stdout_encoding = getattr(sys.stdout, "encoding", None) - - scheme_names = sysconfig.get_scheme_names() - - if "venv" in scheme_names: - self.sysconfig_scheme = "venv" - self.sysconfig_paths = { - i: sysconfig.get_path(i, expand=False, scheme=self.sysconfig_scheme) for i in sysconfig.get_path_names() - } - # we cannot use distutils at all if "venv" exists, distutils don't know it - self.distutils_install = {} - # debian / ubuntu python 3.10 without `python3-distutils` will report - # mangled `local/bin` / etc. names for the default prefix - # intentionally select `posix_prefix` which is the unaltered posix-like paths - elif sys.version_info[:2] == (3, 10) and "deb_system" in scheme_names: - self.sysconfig_scheme = "posix_prefix" - self.sysconfig_paths = { - i: sysconfig.get_path(i, expand=False, scheme=self.sysconfig_scheme) for i in sysconfig.get_path_names() - } - # we cannot use distutils at all if "venv" exists, distutils don't know it - self.distutils_install = {} - else: - self.sysconfig_scheme = None - self.sysconfig_paths = {i: sysconfig.get_path(i, expand=False) for i in sysconfig.get_path_names()} - self.distutils_install = self._distutils_install().copy() - - # https://bugs.python.org/issue22199 - makefile = getattr(sysconfig, "get_makefile_filename", getattr(sysconfig, "_get_makefile_filename", None)) - self.sysconfig = { - k: v - for k, v in [ - # a list of content to store from sysconfig - ("makefile_filename", makefile()), - ] - if k is not None - } - - config_var_keys = set() - for element in self.sysconfig_paths.values(): - config_var_keys.update(k[1:-1] for k in _CONF_VAR_RE.findall(element)) - config_var_keys.add("PYTHONFRAMEWORK") - - self.sysconfig_vars = {i: sysconfig.get_config_var(i or "") for i in config_var_keys} - - if "TCL_LIBRARY" in os.environ: - self.tcl_lib, self.tk_lib = self._get_tcl_tk_libs() - else: - self.tcl_lib, self.tk_lib = None, None - - confs = { - k: (self.system_prefix if v is not None and v.startswith(self.prefix) else v) - for k, v in self.sysconfig_vars.items() - } - self.system_stdlib = self.sysconfig_path("stdlib", confs) - self.system_stdlib_platform = self.sysconfig_path("platstdlib", confs) - self.max_size = getattr(sys, "maxsize", getattr(sys, "maxint", None)) - self._creators = None - - @staticmethod - def _get_tcl_tk_libs(): - """ - Detects the tcl and tk libraries using tkinter. - - This works reliably but spins up tkinter, which is heavy if you don't need it. - """ - tcl_lib, tk_lib = None, None - try: - import tkinter as tk # noqa: PLC0415 - except ImportError: - pass - else: - try: - tcl = tk.Tcl() - tcl_lib = tcl.eval("info library") - - # Try to get TK library path directly first - try: - tk_lib = tcl.eval("set tk_library") - if tk_lib and os.path.isdir(tk_lib): - pass # We found it directly - else: - tk_lib = None # Reset if invalid - except tk.TclError: - tk_lib = None - - # If direct query failed, try constructing the path - if tk_lib is None: - tk_version = tcl.eval("package require Tk") - tcl_parent = os.path.dirname(tcl_lib) - - # Try different version formats - version_variants = [ - tk_version, # Full version like "8.6.12" - ".".join(tk_version.split(".")[:2]), # Major.minor like "8.6" - tk_version.split(".")[0], # Just major like "8" - ] - - for version in version_variants: - tk_lib_path = os.path.join(tcl_parent, f"tk{version}") - if not os.path.isdir(tk_lib_path): - continue - # Validate it's actually a TK directory - if os.path.exists(os.path.join(tk_lib_path, "tk.tcl")): - tk_lib = tk_lib_path - break - - except tk.TclError: - pass - - return tcl_lib, tk_lib - - def _fast_get_system_executable(self): - """Try to get the system executable by just looking at properties.""" - # if we're not in a virtual environment, this is already a system python, so return the original executable - # note we must choose the original and not the pure executable as shim scripts might throw us off - if not (self.real_prefix or (self.base_prefix is not None and self.base_prefix != self.prefix)): - return self.original_executable - - # if this is NOT a virtual environment, can't determine easily, bail out - if self.real_prefix is not None: - return None - - base_executable = getattr(sys, "_base_executable", None) # some platforms may set this to help us - if base_executable is None: # use the saved system executable if present - return None - - # we know we're in a virtual environment, can not be us - if sys.executable == base_executable: - return None - - # We're not in a venv and base_executable exists; use it directly - if os.path.exists(base_executable): - return base_executable - - # Try fallback for POSIX virtual environments - return self._try_posix_fallback_executable(base_executable) - - def _try_posix_fallback_executable(self, base_executable): - """ - Try to find a versioned Python binary as fallback for POSIX virtual environments. - - Python may return "python" because it was invoked from the POSIX virtual environment - however some installs/distributions do not provide a version-less "python" binary in - the system install location (see PEP 394) so try to fallback to a versioned binary. - - Gate this to Python 3.11 as `sys._base_executable` path resolution is now relative to - the 'home' key from pyvenv.cfg which often points to the system install location. - """ - major, minor = self.version_info.major, self.version_info.minor - if self.os != "posix" or (major, minor) < (3, 11): - return None - - # search relative to the directory of sys._base_executable - base_dir = os.path.dirname(base_executable) - candidates = [f"python{major}", f"python{major}.{minor}"] - if self.implementation == "PyPy": - candidates.extend(["pypy", "pypy3", f"pypy{major}", f"pypy{major}.{minor}"]) - - for candidate in candidates: - full_path = os.path.join(base_dir, candidate) - if os.path.exists(full_path): - return full_path - - return None # in this case we just can't tell easily without poking around FS and calling them, bail - - def install_path(self, key): - result = self.distutils_install.get(key) - if result is None: # use sysconfig if sysconfig_scheme is set or distutils is unavailable - # set prefixes to empty => result is relative from cwd - prefixes = self.prefix, self.exec_prefix, self.base_prefix, self.base_exec_prefix - config_var = {k: "" if v in prefixes else v for k, v in self.sysconfig_vars.items()} - result = self.sysconfig_path(key, config_var=config_var).lstrip(os.sep) - return result - - @staticmethod - def _distutils_install(): - # use distutils primarily because that's what pip does - # https://github.com/pypa/pip/blob/main/src/pip/_internal/locations.py#L95 - # note here we don't import Distribution directly to allow setuptools to patch it - with warnings.catch_warnings(): # disable warning for PEP-632 - warnings.simplefilter("ignore") - try: - from distutils import dist # noqa: PLC0415 - from distutils.command.install import SCHEME_KEYS # noqa: PLC0415 - except ImportError: # if removed or not installed ignore - return {} - - d = dist.Distribution({"script_args": "--no-user-cfg"}) # conf files not parsed so they do not hijack paths - if hasattr(sys, "_framework"): - sys._framework = None # disable macOS static paths for framework # noqa: SLF001 - - with warnings.catch_warnings(): # disable warning for PEP-632 - warnings.simplefilter("ignore") - i = d.get_command_obj("install", create=True) - - i.prefix = os.sep # paths generated are relative to prefix that contains the path sep, this makes it relative - i.finalize_options() - return {key: (getattr(i, f"install_{key}")[1:]).lstrip(os.sep) for key in SCHEME_KEYS} - - @property - def version_str(self): - return ".".join(str(i) for i in self.version_info[0:3]) - - @property - def version_release_str(self): - return ".".join(str(i) for i in self.version_info[0:2]) - - @property - def python_name(self): - version_info = self.version_info - return f"python{version_info.major}.{version_info.minor}" - - @property - def is_old_virtualenv(self): - return self.real_prefix is not None - - @property - def is_venv(self): - return self.base_prefix is not None - - def sysconfig_path(self, key, config_var=None, sep=os.sep): - pattern = self.sysconfig_paths.get(key) - if pattern is None: - return "" - if config_var is None: - config_var = self.sysconfig_vars - else: - base = self.sysconfig_vars.copy() - base.update(config_var) - config_var = base - return pattern.format(**config_var).replace("/", sep) - - def creators(self, refresh=False): # noqa: FBT002 - if self._creators is None or refresh is True: - from virtualenv.run.plugin.creators import CreatorSelector # noqa: PLC0415 - - self._creators = CreatorSelector.for_interpreter(self) - return self._creators - - @property - def system_include(self): - path = self.sysconfig_path( - "include", - { - k: (self.system_prefix if v is not None and v.startswith(self.prefix) else v) - for k, v in self.sysconfig_vars.items() - }, - ) - if not os.path.exists(path): # some broken packaging don't respect the sysconfig, fallback to distutils path - # the pattern include the distribution name too at the end, remove that via the parent call - fallback = os.path.join(self.prefix, os.path.dirname(self.install_path("headers"))) - if os.path.exists(fallback): - path = fallback - return path - - @property - def system_prefix(self): - return self.real_prefix or self.base_prefix or self.prefix - - @property - def system_exec_prefix(self): - return self.real_prefix or self.base_exec_prefix or self.exec_prefix - - def __repr__(self) -> str: - return "{}({!r})".format( - self.__class__.__name__, - {k: v for k, v in self.__dict__.items() if not k.startswith("_")}, - ) - - def __str__(self) -> str: - return "{}({})".format( - self.__class__.__name__, - ", ".join( - f"{k}={v}" - for k, v in ( - ("spec", self.spec), - ( - "system" - if self.system_executable is not None and self.system_executable != self.executable - else None, - self.system_executable, - ), - ( - "original" - if self.original_executable not in {self.system_executable, self.executable} - else None, - self.original_executable, - ), - ("exe", self.executable), - ("platform", self.platform), - ("version", repr(self.version)), - ("encoding_fs_io", f"{self.file_system_encoding}-{self.stdout_encoding}"), - ) - if k is not None - ), - ) - - @property - def spec(self): - return "{}{}{}-{}".format( - self.implementation, - ".".join(str(i) for i in self.version_info), - "t" if self.free_threaded else "", - self.architecture, - ) - - @classmethod - def clear_cache(cls, app_data): - # this method is not used by itself, so here and called functions can import stuff locally - from virtualenv.discovery.cached_py_info import clear # noqa: PLC0415 - - clear(app_data) - cls._cache_exe_discovery.clear() - - def satisfies(self, spec, impl_must_match): # noqa: C901, PLR0911 - """Check if a given specification can be satisfied by the this python interpreter instance.""" - if spec.path: - if self.executable == os.path.abspath(spec.path): - return True # if the path is a our own executable path we're done - if not spec.is_abs: - # if path set, and is not our original executable name, this does not match - basename = os.path.basename(self.original_executable) - spec_path = spec.path - if sys.platform == "win32": - basename, suffix = os.path.splitext(basename) - if spec_path.endswith(suffix): - spec_path = spec_path[: -len(suffix)] - if basename != spec_path: - return False - - if ( - impl_must_match - and spec.implementation is not None - and spec.implementation.lower() != self.implementation.lower() - ): - return False - - if spec.architecture is not None and spec.architecture != self.architecture: - return False - - if spec.free_threaded is not None and spec.free_threaded != self.free_threaded: - return False - - for our, req in zip(self.version_info[0:3], (spec.major, spec.minor, spec.micro)): - if req is not None and our is not None and our != req: - return False - return True - - _current_system = None - _current = None - - @classmethod - def current(cls, app_data=None): - """ - This locates the current host interpreter information. This might be different than what we run into in case - the host python has been upgraded from underneath us. - """ # noqa: D205 - if cls._current is None: - cls._current = cls.from_exe(sys.executable, app_data, raise_on_error=True, resolve_to_host=False) - return cls._current - - @classmethod - def current_system(cls, app_data=None) -> PythonInfo: - """ - This locates the current host interpreter information. This might be different than what we run into in case - the host python has been upgraded from underneath us. - """ # noqa: D205 - if cls._current_system is None: - cls._current_system = cls.from_exe(sys.executable, app_data, raise_on_error=True, resolve_to_host=True) - return cls._current_system - - def _to_json(self): - # don't save calculated paths, as these are non primitive types - return json.dumps(self._to_dict(), indent=2) - - def _to_dict(self): - data = {var: (getattr(self, var) if var != "_creators" else None) for var in vars(self)} - - data["version_info"] = data["version_info"]._asdict() # namedtuple to dictionary - return data - - @classmethod - def from_exe( # noqa: PLR0913 - cls, - exe, - app_data=None, - raise_on_error=True, # noqa: FBT002 - ignore_cache=False, # noqa: FBT002 - resolve_to_host=True, # noqa: FBT002 - env=None, - ): - """Given a path to an executable get the python information.""" - # this method is not used by itself, so here and called functions can import stuff locally - from virtualenv.discovery.cached_py_info import from_exe # noqa: PLC0415 - - env = os.environ if env is None else env - proposed = from_exe(cls, app_data, exe, env=env, raise_on_error=raise_on_error, ignore_cache=ignore_cache) - - if isinstance(proposed, PythonInfo) and resolve_to_host: - try: - proposed = proposed._resolve_to_system(app_data, proposed) # noqa: SLF001 - except Exception as exception: - if raise_on_error: - raise - LOGGER.info("ignore %s due cannot resolve system due to %r", proposed.original_executable, exception) - proposed = None - return proposed - - @classmethod - def _from_json(cls, payload): - # the dictionary unroll here is to protect against pypy bug of interpreter crashing - raw = json.loads(payload) - return cls._from_dict(raw.copy()) - - @classmethod - def _from_dict(cls, data): - data["version_info"] = VersionInfo(**data["version_info"]) # restore this to a named tuple structure - result = cls() - result.__dict__ = data.copy() - return result - - @classmethod - def _resolve_to_system(cls, app_data, target): - start_executable = target.executable - prefixes = OrderedDict() - while target.system_executable is None: - prefix = target.real_prefix or target.base_prefix or target.prefix - if prefix in prefixes: - if len(prefixes) == 1: - # if we're linking back to ourselves accept ourselves with a WARNING - LOGGER.info("%r links back to itself via prefixes", target) - target.system_executable = target.executable - break - for at, (p, t) in enumerate(prefixes.items(), start=1): - LOGGER.error("%d: prefix=%s, info=%r", at, p, t) - LOGGER.error("%d: prefix=%s, info=%r", len(prefixes) + 1, prefix, target) - msg = "prefixes are causing a circle {}".format("|".join(prefixes.keys())) - raise RuntimeError(msg) - prefixes[prefix] = target - target = target.discover_exe(app_data, prefix=prefix, exact=False) - if target.executable != target.system_executable: - target = cls.from_exe(target.system_executable, app_data) - target.executable = start_executable - return target - - _cache_exe_discovery = {} # noqa: RUF012 - - def discover_exe(self, app_data, prefix, exact=True, env=None): # noqa: FBT002 - key = prefix, exact - if key in self._cache_exe_discovery and prefix: - LOGGER.debug("discover exe from cache %s - exact %s: %r", prefix, exact, self._cache_exe_discovery[key]) - return self._cache_exe_discovery[key] - LOGGER.debug("discover exe for %s in %s", self, prefix) - # we don't know explicitly here, do some guess work - our executable name should tell - possible_names = self._find_possible_exe_names() - possible_folders = self._find_possible_folders(prefix) - discovered = [] - env = os.environ if env is None else env - for folder in possible_folders: - for name in possible_names: - info = self._check_exe(app_data, folder, name, exact, discovered, env) - if info is not None: - self._cache_exe_discovery[key] = info - return info - if exact is False and discovered: - info = self._select_most_likely(discovered, self) - folders = os.pathsep.join(possible_folders) - self._cache_exe_discovery[key] = info - LOGGER.debug("no exact match found, chosen most similar of %s within base folders %s", info, folders) - return info - msg = "failed to detect {} in {}".format("|".join(possible_names), os.pathsep.join(possible_folders)) - raise RuntimeError(msg) - - def _check_exe(self, app_data, folder, name, exact, discovered, env): # noqa: PLR0913 - exe_path = os.path.join(folder, name) - if not os.path.exists(exe_path): - return None - info = self.from_exe(exe_path, app_data, resolve_to_host=False, raise_on_error=False, env=env) - if info is None: # ignore if for some reason we can't query - return None - for item in ["implementation", "architecture", "version_info"]: - found = getattr(info, item) - searched = getattr(self, item) - if found != searched: - if item == "version_info": - found, searched = ".".join(str(i) for i in found), ".".join(str(i) for i in searched) - executable = info.executable - LOGGER.debug("refused interpreter %s because %s differs %s != %s", executable, item, found, searched) - if exact is False: - discovered.append(info) - break - else: - return info - return None - - @staticmethod - def _select_most_likely(discovered, target): - # no exact match found, start relaxing our requirements then to facilitate system package upgrades that - # could cause this (when using copy strategy of the host python) - def sort_by(info): - # we need to setup some priority of traits, this is as follows: - # implementation, major, minor, micro, architecture, tag, serial - matches = [ - info.implementation == target.implementation, - info.version_info.major == target.version_info.major, - info.version_info.minor == target.version_info.minor, - info.architecture == target.architecture, - info.version_info.micro == target.version_info.micro, - info.version_info.releaselevel == target.version_info.releaselevel, - info.version_info.serial == target.version_info.serial, - ] - return sum((1 << pos if match else 0) for pos, match in enumerate(reversed(matches))) - - sorted_discovered = sorted(discovered, key=sort_by, reverse=True) # sort by priority in decreasing order - return sorted_discovered[0] - - def _find_possible_folders(self, inside_folder): - candidate_folder = OrderedDict() - executables = OrderedDict() - executables[os.path.realpath(self.executable)] = None - executables[self.executable] = None - executables[os.path.realpath(self.original_executable)] = None - executables[self.original_executable] = None - for exe in executables: - base = os.path.dirname(exe) - # following path pattern of the current - if base.startswith(self.prefix): - relative = base[len(self.prefix) :] - candidate_folder[f"{inside_folder}{relative}"] = None - - # or at root level - candidate_folder[inside_folder] = None - return [i for i in candidate_folder if os.path.exists(i)] - - def _find_possible_exe_names(self): - name_candidate = OrderedDict() - for name in self._possible_base(): - for at in (3, 2, 1, 0): - version = ".".join(str(i) for i in self.version_info[:at]) - mods = [""] - if self.free_threaded: - mods.append("t") - for mod in mods: - for arch in [f"-{self.architecture}", ""]: - for ext in EXTENSIONS: - candidate = f"{name}{version}{mod}{arch}{ext}" - name_candidate[candidate] = None - return list(name_candidate.keys()) - - def _possible_base(self): - possible_base = OrderedDict() - basename = os.path.splitext(os.path.basename(self.executable))[0].rstrip(digits) - possible_base[basename] = None - possible_base[self.implementation] = None - # python is always the final option as in practice is used by multiple implementation as exe name - if "python" in possible_base: - del possible_base["python"] - possible_base["python"] = None - for base in possible_base: - lower = base.lower() - yield lower - from virtualenv.info import fs_is_case_sensitive # noqa: PLC0415 - - if fs_is_case_sensitive(): - if base != lower: - yield base - upper = base.upper() - if upper != base: - yield upper - - -if __name__ == "__main__": - # dump a JSON representation of the current python - - argv = sys.argv[1:] - - if len(argv) >= 1: - start_cookie = argv[0] - argv = argv[1:] - else: - start_cookie = "" - - if len(argv) >= 1: - end_cookie = argv[0] - argv = argv[1:] - else: - end_cookie = "" - - sys.argv = sys.argv[:1] + argv +from python_discovery import PythonInfo - info = PythonInfo()._to_json() # noqa: SLF001 - sys.stdout.write("".join((start_cookie[::-1], info, end_cookie[::-1]))) +__all__ = [ + "PythonInfo", +] diff --git a/src/virtualenv/discovery/py_spec.py b/src/virtualenv/discovery/py_spec.py index d8519c23d..c407f6e0f 100644 --- a/src/virtualenv/discovery/py_spec.py +++ b/src/virtualenv/discovery/py_spec.py @@ -1,128 +1,8 @@ -"""A Python specification is an abstract requirement definition of an interpreter.""" +"""Backward-compatibility re-export — use ``python_discovery.PythonSpec`` directly.""" from __future__ import annotations -import os -import re - -PATTERN = re.compile(r"^(?P[a-zA-Z]+)?(?P[0-9.]+)?(?Pt)?(?:-(?P32|64))?$") - - -class PythonSpec: - """Contains specification about a Python Interpreter.""" - - def __init__( # noqa: PLR0913 - self, - str_spec: str, - implementation: str | None, - major: int | None, - minor: int | None, - micro: int | None, - architecture: int | None, - path: str | None, - *, - free_threaded: bool | None = None, - ) -> None: - self.str_spec = str_spec - self.implementation = implementation - self.major = major - self.minor = minor - self.micro = micro - self.free_threaded = free_threaded - self.architecture = architecture - self.path = path - - @classmethod - def from_string_spec(cls, string_spec: str): # noqa: C901, PLR0912 - impl, major, minor, micro, threaded, arch, path = None, None, None, None, None, None, None - if os.path.isabs(string_spec): # noqa: PLR1702 - path = string_spec - else: - ok = False - match = re.match(PATTERN, string_spec) - if match: - - def _int_or_none(val): - return None if val is None else int(val) - - try: - groups = match.groupdict() - version = groups["version"] - if version is not None: - versions = tuple(int(i) for i in version.split(".") if i) - if len(versions) > 3: # noqa: PLR2004 - raise ValueError # noqa: TRY301 - if len(versions) == 3: # noqa: PLR2004 - major, minor, micro = versions - elif len(versions) == 2: # noqa: PLR2004 - major, minor = versions - elif len(versions) == 1: - version_data = versions[0] - major = int(str(version_data)[0]) # first digit major - if version_data > 9: # noqa: PLR2004 - minor = int(str(version_data)[1:]) - threaded = bool(groups["threaded"]) - ok = True - except ValueError: - pass - else: - impl = groups["impl"] - if impl in {"py", "python"}: - impl = None - arch = _int_or_none(groups["arch"]) - - if not ok: - path = string_spec - - return cls(string_spec, impl, major, minor, micro, arch, path, free_threaded=threaded) - - def generate_re(self, *, windows: bool) -> re.Pattern: - """Generate a regular expression for matching against a filename.""" - version = r"{}(\.{}(\.{})?)?".format( - *(r"\d+" if v is None else v for v in (self.major, self.minor, self.micro)) - ) - impl = "python" if self.implementation is None else f"python|{re.escape(self.implementation)}" - mod = "t?" if self.free_threaded else "" - suffix = r"\.exe" if windows else "" - version_conditional = ( - "?" - # Windows Python executables are almost always unversioned - if windows - # Spec is an empty string - or self.major is None - else "" - ) - # Try matching `direct` first, so the `direct` group is filled when possible. - return re.compile( - rf"(?P{impl})(?P{version}{mod}){version_conditional}{suffix}$", - flags=re.IGNORECASE, - ) - - @property - def is_abs(self): - return self.path is not None and os.path.isabs(self.path) - - def satisfies(self, spec): - """Called when there's a candidate metadata spec to see if compatible - e.g. PEP-514 on Windows.""" - if spec.is_abs and self.is_abs and self.path != spec.path: - return False - if spec.implementation is not None and spec.implementation.lower() != self.implementation.lower(): - return False - if spec.architecture is not None and spec.architecture != self.architecture: - return False - if spec.free_threaded is not None and spec.free_threaded != self.free_threaded: - return False - - for our, req in zip((self.major, self.minor, self.micro), (spec.major, spec.minor, spec.micro)): - if req is not None and our is not None and our != req: - return False - return True - - def __repr__(self) -> str: - name = type(self).__name__ - params = "implementation", "major", "minor", "micro", "architecture", "path", "free_threaded" - return f"{name}({', '.join(f'{k}={getattr(self, k)}' for k in params if getattr(self, k) is not None)})" - +from python_discovery import PythonSpec __all__ = [ "PythonSpec", diff --git a/src/virtualenv/discovery/windows/__init__.py b/src/virtualenv/discovery/windows/__init__.py deleted file mode 100644 index b7206406a..000000000 --- a/src/virtualenv/discovery/windows/__init__.py +++ /dev/null @@ -1,47 +0,0 @@ -from __future__ import annotations - -from virtualenv.discovery.py_info import PythonInfo -from virtualenv.discovery.py_spec import PythonSpec - -from .pep514 import discover_pythons - -# Map of well-known organizations (as per PEP 514 Company Windows Registry key part) versus Python implementation -_IMPLEMENTATION_BY_ORG = { - "ContinuumAnalytics": "CPython", - "PythonCore": "CPython", -} - - -class Pep514PythonInfo(PythonInfo): - """A Python information acquired from PEP-514.""" - - -def propose_interpreters(spec, cache_dir, env): - # see if PEP-514 entries are good - - # start with higher python versions in an effort to use the latest version available - # and prefer PythonCore over conda pythons (as virtualenv is mostly used by non conda tools) - existing = list(discover_pythons()) - existing.sort( - key=lambda i: (*tuple(-1 if j is None else j for j in i[1:4]), 1 if i[0] == "PythonCore" else 0), - reverse=True, - ) - - for name, major, minor, arch, threaded, exe, _ in existing: - # Map well-known/most common organizations to a Python implementation, use the org name as a fallback for - # backwards compatibility. - implementation = _IMPLEMENTATION_BY_ORG.get(name, name) - - # Pre-filtering based on Windows Registry metadata, for CPython only - skip_pre_filter = implementation.lower() != "cpython" - registry_spec = PythonSpec(None, implementation, major, minor, None, arch, exe, free_threaded=threaded) - if skip_pre_filter or registry_spec.satisfies(spec): - interpreter = Pep514PythonInfo.from_exe(exe, cache_dir, env=env, raise_on_error=False) - if interpreter is not None and interpreter.satisfies(spec, impl_must_match=True): - yield interpreter # Final filtering/matching using interpreter metadata - - -__all__ = [ - "Pep514PythonInfo", - "propose_interpreters", -] diff --git a/src/virtualenv/discovery/windows/pep514.py b/src/virtualenv/discovery/windows/pep514.py deleted file mode 100644 index a75dad36d..000000000 --- a/src/virtualenv/discovery/windows/pep514.py +++ /dev/null @@ -1,165 +0,0 @@ -"""Implement https://www.python.org/dev/peps/pep-0514/ to discover interpreters - Windows only.""" - -from __future__ import annotations - -import os -import re -import winreg -from logging import basicConfig, getLogger - -LOGGER = getLogger(__name__) - - -def enum_keys(key): - at = 0 - while True: - try: - yield winreg.EnumKey(key, at) - except OSError: - break - at += 1 - - -def get_value(key, value_name): - try: - return winreg.QueryValueEx(key, value_name)[0] - except OSError: - return None - - -def discover_pythons(): - for hive, hive_name, key, flags, default_arch in [ - (winreg.HKEY_CURRENT_USER, "HKEY_CURRENT_USER", r"Software\Python", 0, 64), - (winreg.HKEY_LOCAL_MACHINE, "HKEY_LOCAL_MACHINE", r"Software\Python", winreg.KEY_WOW64_64KEY, 64), - (winreg.HKEY_LOCAL_MACHINE, "HKEY_LOCAL_MACHINE", r"Software\Python", winreg.KEY_WOW64_32KEY, 32), - ]: - yield from process_set(hive, hive_name, key, flags, default_arch) - - -def process_set(hive, hive_name, key, flags, default_arch): - try: - with winreg.OpenKeyEx(hive, key, 0, winreg.KEY_READ | flags) as root_key: - for company in enum_keys(root_key): - if company == "PyLauncher": # reserved - continue - yield from process_company(hive_name, company, root_key, default_arch) - except OSError: - pass - - -def process_company(hive_name, company, root_key, default_arch): - with winreg.OpenKeyEx(root_key, company) as company_key: - for tag in enum_keys(company_key): - spec = process_tag(hive_name, company, company_key, tag, default_arch) - if spec is not None: - yield spec - - -def process_tag(hive_name, company, company_key, tag, default_arch): - with winreg.OpenKeyEx(company_key, tag) as tag_key: - version = load_version_data(hive_name, company, tag, tag_key) - if version is not None: # if failed to get version bail - major, minor, _ = version - arch = load_arch_data(hive_name, company, tag, tag_key, default_arch) - if arch is not None: - exe_data = load_exe(hive_name, company, company_key, tag) - if exe_data is not None: - exe, args = exe_data - threaded = load_threaded(hive_name, company, tag, tag_key) - return company, major, minor, arch, threaded, exe, args - return None - return None - return None - - -def load_exe(hive_name, company, company_key, tag): - key_path = f"{hive_name}/{company}/{tag}" - try: - with winreg.OpenKeyEx(company_key, rf"{tag}\InstallPath") as ip_key, ip_key: - exe = get_value(ip_key, "ExecutablePath") - if exe is None: - ip = get_value(ip_key, None) - if ip is None: - msg(key_path, "no ExecutablePath or default for it") - - else: - exe = os.path.join(ip, "python.exe") - if exe is not None and os.path.exists(exe): - args = get_value(ip_key, "ExecutableArguments") - return exe, args - msg(key_path, f"could not load exe with value {exe}") - except OSError: - msg(f"{key_path}/InstallPath", "missing") - return None - - -def load_arch_data(hive_name, company, tag, tag_key, default_arch): - arch_str = get_value(tag_key, "SysArchitecture") - if arch_str is not None: - key_path = f"{hive_name}/{company}/{tag}/SysArchitecture" - try: - return parse_arch(arch_str) - except ValueError as sys_arch: - msg(key_path, sys_arch) - return default_arch - - -def parse_arch(arch_str): - if isinstance(arch_str, str): - match = re.match(r"^(\d+)bit$", arch_str) - if match: - return int(next(iter(match.groups()))) - error = f"invalid format {arch_str}" - else: - error = f"arch is not string: {arch_str!r}" - raise ValueError(error) - - -def load_version_data(hive_name, company, tag, tag_key): - for candidate, key_path in [ - (get_value(tag_key, "SysVersion"), f"{hive_name}/{company}/{tag}/SysVersion"), - (tag, f"{hive_name}/{company}/{tag}"), - ]: - if candidate is not None: - try: - return parse_version(candidate) - except ValueError as sys_version: - msg(key_path, sys_version) - return None - - -def parse_version(version_str): - if isinstance(version_str, str): - match = re.match(r"^(\d+)(?:\.(\d+))?(?:\.(\d+))?$", version_str) - if match: - return tuple(int(i) if i is not None else None for i in match.groups()) - error = f"invalid format {version_str}" - else: - error = f"version is not string: {version_str!r}" - raise ValueError(error) - - -def load_threaded(hive_name, company, tag, tag_key): - display_name = get_value(tag_key, "DisplayName") - if display_name is not None: - if isinstance(display_name, str): - if "freethreaded" in display_name.lower(): - return True - else: - key_path = f"{hive_name}/{company}/{tag}/DisplayName" - msg(key_path, f"display name is not string: {display_name!r}") - return bool(re.match(r"^\d+(\.\d+){0,2}t$", tag, flags=re.IGNORECASE)) - - -def msg(path, what): - LOGGER.warning("PEP-514 violation in Windows Registry at %s error: %s", path, what) - - -def _run(): - basicConfig() - interpreters = [repr(spec) for spec in discover_pythons()] - print("\n".join(sorted(interpreters))) # noqa: T201 - - -if __name__ == "__main__": - _run() diff --git a/src/virtualenv/info.py b/src/virtualenv/info.py index e3542a7e2..1f255cad2 100644 --- a/src/virtualenv/info.py +++ b/src/virtualenv/info.py @@ -9,6 +9,7 @@ IMPLEMENTATION = platform.python_implementation() IS_PYPY = IMPLEMENTATION == "PyPy" IS_GRAALPY = IMPLEMENTATION == "GraalVM" +IS_RUSTPYTHON = IMPLEMENTATION == "RustPython" IS_CPYTHON = IMPLEMENTATION == "CPython" IS_WIN = sys.platform == "win32" IS_MAC_ARM64 = sys.platform == "darwin" and platform.machine() == "arm64" @@ -18,8 +19,8 @@ LOGGER = logging.getLogger(__name__) -def fs_is_case_sensitive(): - global _FS_CASE_SENSITIVE # noqa: PLW0603 +def fs_is_case_sensitive() -> bool: + global _FS_CASE_SENSITIVE # ruff:ignore[global-statement] if _FS_CASE_SENSITIVE is None: with tempfile.NamedTemporaryFile(prefix="TmP") as tmp_file: @@ -28,8 +29,8 @@ def fs_is_case_sensitive(): return _FS_CASE_SENSITIVE -def fs_supports_symlink(): - global _CAN_SYMLINK # noqa: PLW0603 +def fs_supports_symlink() -> bool: + global _CAN_SYMLINK # ruff:ignore[global-statement] if _CAN_SYMLINK is None: can = False @@ -61,6 +62,7 @@ def fs_path_id(path: str) -> str: "IS_GRAALPY", "IS_MAC_ARM64", "IS_PYPY", + "IS_RUSTPYTHON", "IS_WIN", "IS_ZIPAPP", "ROOT", diff --git a/src/virtualenv/py.typed b/src/virtualenv/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/src/virtualenv/report.py b/src/virtualenv/report.py index c9682a8f6..b7621f902 100644 --- a/src/virtualenv/report.py +++ b/src/virtualenv/report.py @@ -16,7 +16,7 @@ LOGGER = logging.getLogger() -def setup_report(verbosity, show_pid=False): # noqa: FBT002 +def setup_report(verbosity: int, show_pid: bool = False) -> int: # ruff:ignore[boolean-default-value-positional-argument] _clean_handlers(LOGGER) verbosity = min(verbosity, MAX_LEVEL) # pragma: no cover level = LEVELS[verbosity] @@ -38,7 +38,7 @@ def setup_report(verbosity, show_pid=False): # noqa: FBT002 return verbosity -def _clean_handlers(log): +def _clean_handlers(log: logging.Logger) -> None: for log_handler in list(log.handlers): # remove handlers of libraries log.removeHandler(log_handler) diff --git a/src/virtualenv/run/__init__.py b/src/virtualenv/run/__init__.py index 03190502b..9788ebd38 100644 --- a/src/virtualenv/run/__init__.py +++ b/src/virtualenv/run/__init__.py @@ -3,9 +3,10 @@ import logging import os from functools import partial +from typing import TYPE_CHECKING from virtualenv.app_data import make_app_data -from virtualenv.config.cli.parser import VirtualEnvConfigParser +from virtualenv.config.cli.parser import VirtualEnvConfigParser, VirtualEnvOptions from virtualenv.report import LEVELS, setup_report from virtualenv.run.session import Session from virtualenv.seed.wheels.periodic_update import manual_upgrade @@ -16,16 +17,28 @@ from .plugin.discovery import get_discover from .plugin.seeders import SeederSelector +if TYPE_CHECKING: + from collections.abc import MutableMapping -def cli_run(args, options=None, setup_logging=True, env=None): # noqa: FBT002 - """ - Create a virtual environment given some command line interface arguments. + from .plugin.base import ComponentBuilder + + +def cli_run( + args: list[str], + options: VirtualEnvOptions | None = None, + setup_logging: bool = True, # ruff:ignore[boolean-default-value-positional-argument] + env: MutableMapping[str, str] | None = None, +) -> Session: + """Create a virtual environment given some command line interface arguments. :param args: the command line arguments :param options: passing in a ``VirtualEnvOptions`` object allows return of the parsed options :param setup_logging: ``True`` if setup logging handlers, ``False`` to use handlers already registered :param env: environment variables to use - :return: the session object of the creation (its structure for now is experimental and might change on short notice) + + :returns: the session object of the creation (its structure for now is experimental and might change on short + notice) + """ env = os.environ if env is None else env of_session = session_via_cli(args, options, setup_logging, env) @@ -34,33 +47,47 @@ def cli_run(args, options=None, setup_logging=True, env=None): # noqa: FBT002 return of_session -def session_via_cli(args, options=None, setup_logging=True, env=None): # noqa: FBT002 - """ - Create a virtualenv session (same as cli_run, but this does not perform the creation). Use this if you just want to - query what the virtual environment would look like, but not actually create it. +def session_via_cli( + args: list[str], + options: VirtualEnvOptions | None = None, + setup_logging: bool = True, # ruff:ignore[boolean-default-value-positional-argument] + env: MutableMapping[str, str] | None = None, +) -> Session: + """Create a virtualenv session (same as cli_run, but this does not perform the creation). Use this if you just want to query what the virtual environment would look like, but not actually create it. :param args: the command line arguments :param options: passing in a ``VirtualEnvOptions`` object allows return of the parsed options :param setup_logging: ``True`` if setup logging handlers, ``False`` to use handlers already registered :param env: environment variables to use - :return: the session object of the creation (its structure for now is experimental and might change on short notice) - """ # noqa: D205 + + :returns: the session object of the creation (its structure for now is experimental and might change on short + notice) + + """ env = os.environ if env is None else env parser, elements = build_parser(args, options, setup_logging, env) - options = parser.parse_args(args) - options.py_version = parser._interpreter.version_info # noqa: SLF001 - creator, seeder, activators = tuple(e.create(options) for e in elements) # create types + options = parser.parse_args(args) # ty: ignore[invalid-assignment] + options.py_version = parser._interpreter.version_info # ruff:ignore[private-member-access] # ty: ignore[invalid-assignment, unresolved-attribute] + creator, seeder, activators = tuple( + e.create(options) # ty: ignore[invalid-argument-type] + for e in elements + ) # create types return Session( - options.verbosity, - options.app_data, - parser._interpreter, # noqa: SLF001 - creator, - seeder, - activators, + options.verbosity, # ty: ignore[unresolved-attribute, invalid-argument-type] + options.app_data, # ty: ignore[unresolved-attribute] + parser._interpreter, # ruff:ignore[private-member-access] # ty: ignore[invalid-argument-type] + creator, # ty: ignore[invalid-argument-type] + seeder, # ty: ignore[invalid-argument-type] + activators, # ty: ignore[invalid-argument-type] ) -def build_parser(args=None, options=None, setup_logging=True, env=None): # noqa: FBT002 +def build_parser( + args: list[str] | None = None, + options: VirtualEnvOptions | None = None, + setup_logging: bool = True, # ruff:ignore[boolean-default-value-positional-argument] + env: MutableMapping[str, str] | None = None, +) -> tuple[VirtualEnvConfigParser, list[ComponentBuilder]]: parser = VirtualEnvConfigParser(options, os.environ if env is None else env) add_version_flag(parser) parser.add_argument( @@ -75,11 +102,11 @@ def build_parser(args=None, options=None, setup_logging=True, env=None): # noqa handle_extra_commands(options) discover = get_discover(parser, args) - parser._interpreter = interpreter = discover.interpreter # noqa: SLF001 + parser._interpreter = interpreter = discover.interpreter # ruff:ignore[private-member-access] if interpreter is None: msg = f"failed to find interpreter for {discover}" raise RuntimeError(msg) - elements = [ + elements: list[ComponentBuilder] = [ CreatorSelector(interpreter, parser), SeederSelector(interpreter, parser), ActivationSelector(interpreter, parser), @@ -91,18 +118,20 @@ def build_parser(args=None, options=None, setup_logging=True, env=None): # noqa return parser, elements -def build_parser_only(args=None): +def build_parser_only(args: list[str] | None = None) -> VirtualEnvConfigParser: """Used to provide a parser for the doc generation.""" return build_parser(args)[0] -def handle_extra_commands(options): +def handle_extra_commands(options: VirtualEnvOptions) -> None: if options.upgrade_embed_wheels: result = manual_upgrade(options.app_data, options.env) raise SystemExit(result) -def load_app_data(args, parser, options): +def load_app_data( + args: list[str] | None, parser: VirtualEnvConfigParser, options: VirtualEnvOptions | None +) -> VirtualEnvOptions: parser.add_argument( "--read-only-app-data", action="store_true", @@ -133,8 +162,8 @@ def load_app_data(args, parser, options): return options -def add_version_flag(parser): - import virtualenv # noqa: PLC0415 +def add_version_flag(parser: VirtualEnvConfigParser) -> None: + import virtualenv # ruff:ignore[import-outside-top-level] parser.add_argument( "--version", @@ -144,7 +173,7 @@ def add_version_flag(parser): ) -def _do_report_setup(parser, args, setup_logging): +def _do_report_setup(parser: VirtualEnvConfigParser, args: list[str] | None, setup_logging: bool) -> None: level_map = ", ".join(f"{logging.getLevelName(line)}={c}" for c, line in sorted(LEVELS.items())) msg = "verbosity = verbose - quiet, default {}, mapping => {}" verbosity_group = parser.add_argument_group( @@ -159,7 +188,7 @@ def _do_report_setup(parser, args, setup_logging): return option, _ = parser.parse_known_args(args) if setup_logging: - setup_report(option.verbosity) + setup_report(option.verbosity) # ty: ignore[invalid-argument-type] __all__ = [ diff --git a/src/virtualenv/run/plugin/activators.py b/src/virtualenv/run/plugin/activators.py index a0e866948..15d90e215 100644 --- a/src/virtualenv/run/plugin/activators.py +++ b/src/virtualenv/run/plugin/activators.py @@ -2,21 +2,32 @@ from argparse import ArgumentTypeError from collections import OrderedDict +from typing import TYPE_CHECKING from .base import ComponentBuilder +if TYPE_CHECKING: + from collections.abc import Sequence + + from python_discovery import PythonInfo + + from virtualenv.activation.activator import Activator + from virtualenv.config.cli.parser import VirtualEnvConfigParser, VirtualEnvOptions + class ActivationSelector(ComponentBuilder): - def __init__(self, interpreter, parser) -> None: + def __init__(self, interpreter: PythonInfo, parser: VirtualEnvConfigParser) -> None: self.default = None possible = OrderedDict( - (k, v) for k, v in self.options("virtualenv.activate").items() if v.supports(interpreter) + (k, v) + for k, v in self.options("virtualenv.activate").items() + if v.supports(interpreter) # ty: ignore[unresolved-attribute] ) super().__init__(interpreter, parser, "activators", possible) self.parser.description = "options for activation scripts" self.active = None - def add_selector_arg_parse(self, name, choices): + def add_selector_arg_parse(self, name: str, choices: Sequence[str]) -> None: self.default = ",".join(choices) self.parser.add_argument( f"--{name}", @@ -27,7 +38,7 @@ def add_selector_arg_parse(self, name, choices): type=self._extract_activators, ) - def _extract_activators(self, entered_str): + def _extract_activators(self, entered_str: str) -> list[str]: elements = [e.strip() for e in entered_str.split(",") if e.strip()] missing = [e for e in elements if e not in self.possible] if missing: @@ -35,7 +46,7 @@ def _extract_activators(self, entered_str): raise ArgumentTypeError(msg) return elements - def handle_selected_arg_parse(self, options): + def handle_selected_arg_parse(self, options: VirtualEnvOptions) -> None: # ty: ignore[invalid-method-override] selected_activators = ( self._extract_activators(self.default) if options.activators is self.default else options.activators ) @@ -51,9 +62,10 @@ def handle_selected_arg_parse(self, options): default=None, ) for activator in self.active.values(): - activator.add_parser_arguments(self.parser, self.interpreter) + activator.add_parser_arguments(self.parser, self.interpreter) # ty: ignore[unresolved-attribute] - def create(self, options): + def create(self, options: VirtualEnvOptions) -> list[Activator]: + assert self.active is not None # ruff:ignore[assert] # Set by handle_selected_arg_parse return [activator_class(options) for activator_class in self.active.values()] diff --git a/src/virtualenv/run/plugin/base.py b/src/virtualenv/run/plugin/base.py index 97c4792ec..1f500d282 100644 --- a/src/virtualenv/run/plugin/base.py +++ b/src/virtualenv/run/plugin/base.py @@ -3,8 +3,14 @@ import sys from collections import OrderedDict from importlib.metadata import entry_points +from typing import TYPE_CHECKING -importlib_metadata_version = () +if TYPE_CHECKING: + from collections.abc import Sequence + + from python_discovery import PythonInfo + + from virtualenv.config.cli.parser import VirtualEnvConfigParser, VirtualEnvOptions class PluginLoader: @@ -12,20 +18,28 @@ class PluginLoader: _ENTRY_POINTS = None @classmethod - def entry_points_for(cls, key): - if sys.version_info >= (3, 10) or importlib_metadata_version >= (3, 6): - return OrderedDict((e.name, e.load()) for e in cls.entry_points().select(group=key)) - return OrderedDict((e.name, e.load()) for e in cls.entry_points().get(key, {})) + def entry_points_for(cls, key: str) -> OrderedDict[str, type]: + if sys.version_info >= (3, 10): + selected = list(cls.entry_points().select(group=key)) # ty: ignore[unresolved-attribute] + else: + selected = list(cls.entry_points().get(key, [])) # ty: ignore[unresolved-attribute] + # Third-party packages may register entry points with the same name as virtualenv's + # built-ins (e.g. xonsh's own `virtualenv.activate.xonsh`). Sort so built-ins are + # inserted last into the OrderedDict, making them win on name collision. + selected.sort(key=lambda e: e.value.startswith("virtualenv.")) + return OrderedDict((e.name, e.load()) for e in selected) @staticmethod - def entry_points(): + def entry_points() -> object: if PluginLoader._ENTRY_POINTS is None: PluginLoader._ENTRY_POINTS = entry_points() return PluginLoader._ENTRY_POINTS class ComponentBuilder(PluginLoader): - def __init__(self, interpreter, parser, name, possible) -> None: + def __init__( + self, interpreter: PythonInfo, parser: VirtualEnvConfigParser, name: str, possible: dict[str, type] + ) -> None: self.interpreter = interpreter self.name = name self._impl_class = None @@ -34,15 +48,15 @@ def __init__(self, interpreter, parser, name, possible) -> None: self.add_selector_arg_parse(name, list(self.possible)) @classmethod - def options(cls, key): + def options(cls, key: str) -> OrderedDict[str, type]: if cls._OPTIONS is None: cls._OPTIONS = cls.entry_points_for(key) return cls._OPTIONS - def add_selector_arg_parse(self, name, choices): + def add_selector_arg_parse(self, name: str, choices: Sequence[str]) -> None: raise NotImplementedError - def handle_selected_arg_parse(self, options): + def handle_selected_arg_parse(self, options: VirtualEnvOptions) -> str: selected = getattr(options, self.name) if selected not in self.possible: msg = f"No implementation for {self.interpreter}" @@ -51,11 +65,13 @@ def handle_selected_arg_parse(self, options): self.populate_selected_argparse(selected, options.app_data) return selected - def populate_selected_argparse(self, selected, app_data): + def populate_selected_argparse(self, selected: str, app_data: object) -> None: self.parser.description = f"options for {self.name} {selected}" - self._impl_class.add_parser_arguments(self.parser, self.interpreter, app_data) + assert self._impl_class is not None # ruff:ignore[assert] # Set by handle_selected_arg_parse + self._impl_class.add_parser_arguments(self.parser, self.interpreter, app_data) # ty: ignore[unresolved-attribute] - def create(self, options): + def create(self, options: VirtualEnvOptions) -> object: + assert self._impl_class is not None # ruff:ignore[assert] # Set by handle_selected_arg_parse return self._impl_class(options, self.interpreter) diff --git a/src/virtualenv/run/plugin/creators.py b/src/virtualenv/run/plugin/creators.py index 6bb11845d..748196b27 100644 --- a/src/virtualenv/run/plugin/creators.py +++ b/src/virtualenv/run/plugin/creators.py @@ -9,6 +9,11 @@ from .base import ComponentBuilder if TYPE_CHECKING: + from collections.abc import Sequence + + from python_discovery import PythonInfo + + from virtualenv.config.cli.parser import VirtualEnvConfigParser, VirtualEnvOptions from virtualenv.create.creator import Creator, CreatorMeta @@ -20,19 +25,19 @@ class CreatorInfo(NamedTuple): class CreatorSelector(ComponentBuilder): - def __init__(self, interpreter, parser) -> None: + def __init__(self, interpreter: PythonInfo, parser: VirtualEnvConfigParser) -> None: creators, self.key_to_meta, self.describe, self.builtin_key = self.for_interpreter(interpreter) - super().__init__(interpreter, parser, "creator", creators) + super().__init__(interpreter, parser, "creator", creators) # ty: ignore[invalid-argument-type] @classmethod - def for_interpreter(cls, interpreter): + def for_interpreter(cls, interpreter: PythonInfo) -> CreatorInfo: key_to_class, key_to_meta, builtin_key, describe = OrderedDict(), {}, None, None errors = defaultdict(list) for key, creator_class in cls.options("virtualenv.create").items(): if key == "builtin": msg = "builtin creator is a reserved name" raise RuntimeError(msg) - meta = creator_class.can_create(interpreter) + meta = creator_class.can_create(interpreter) # ty: ignore[unresolved-attribute] if meta: if meta.error: errors[meta.error].append(creator_class) @@ -55,10 +60,10 @@ def for_interpreter(cls, interpreter): key_to_class=key_to_class, key_to_meta=key_to_meta, describe=describe, - builtin_key=builtin_key, + builtin_key=builtin_key or "", ) - def add_selector_arg_parse(self, name, choices): + def add_selector_arg_parse(self, name: str, choices: Sequence[str]) -> None: # prefer the built-in venv if present, otherwise fallback to first defined type choices = sorted(choices, key=lambda a: 0 if a == "builtin" else 1) default_value = self._get_default(choices) @@ -71,18 +76,20 @@ def add_selector_arg_parse(self, name, choices): ) @staticmethod - def _get_default(choices): + def _get_default(choices: list[str]) -> str: return next(iter(choices)) - def populate_selected_argparse(self, selected, app_data): + def populate_selected_argparse(self, selected: str, app_data: object) -> None: self.parser.description = f"options for {self.name} {selected}" - self._impl_class.add_parser_arguments(self.parser, self.interpreter, self.key_to_meta[selected], app_data) + assert self._impl_class is not None # ruff:ignore[assert] # Set by handle_selected_arg_parse + self._impl_class.add_parser_arguments(self.parser, self.interpreter, self.key_to_meta[selected], app_data) # ty: ignore[unresolved-attribute] - def create(self, options): + def create(self, options: VirtualEnvOptions) -> Creator: options.meta = self.key_to_meta[getattr(options, self.name)] + assert self._impl_class is not None # ruff:ignore[assert] # Set by handle_selected_arg_parse if not issubclass(self._impl_class, Describe): - options.describe = self.describe(options, self.interpreter) - return super().create(options) + options.describe = self.describe(options, self.interpreter) # ty: ignore[call-non-callable, invalid-argument-type] + return super().create(options) # ty: ignore[invalid-return-type] __all__ = [ diff --git a/src/virtualenv/run/plugin/discovery.py b/src/virtualenv/run/plugin/discovery.py index 5e8b2392f..837a3bcd7 100644 --- a/src/virtualenv/run/plugin/discovery.py +++ b/src/virtualenv/run/plugin/discovery.py @@ -1,13 +1,19 @@ from __future__ import annotations +from typing import TYPE_CHECKING + from .base import PluginLoader +if TYPE_CHECKING: + from virtualenv.config.cli.parser import VirtualEnvConfigParser + from virtualenv.discovery.discover import Discover + class Discovery(PluginLoader): """Discovery plugins.""" -def get_discover(parser, args): +def get_discover(parser: VirtualEnvConfigParser, args: list[str] | None) -> Discover: discover_types = Discovery.entry_points_for("virtualenv.discovery") discovery_parser = parser.add_argument_group( title="discovery", @@ -29,13 +35,22 @@ def get_discover(parser, args): help="interpreter discovery method", ) options, _ = parser.parse_known_args(args) - discover_class = discover_types[options.discovery] - discover_class.add_parser_arguments(discovery_parser) + discovery = options.discovery + if discovery not in discover_types: + available = ", ".join(sorted(discover_types)) + msg = ( + f"discovery {discovery!r} is not available. " + f"Available discovery methods: {available}. " + f"Is the plugin installed?" + ) + raise RuntimeError(msg) + discover_class = discover_types[discovery] + discover_class.add_parser_arguments(discovery_parser) # ty: ignore[unresolved-attribute] options, _ = parser.parse_known_args(args, namespace=options) return discover_class(options) -def _get_default_discovery(discover_types): +def _get_default_discovery(discover_types: dict[str, type]) -> list[str]: return list(discover_types.keys()) diff --git a/src/virtualenv/run/plugin/seeders.py b/src/virtualenv/run/plugin/seeders.py index b1da34c5d..4723aa9e2 100644 --- a/src/virtualenv/run/plugin/seeders.py +++ b/src/virtualenv/run/plugin/seeders.py @@ -1,14 +1,24 @@ from __future__ import annotations +from typing import TYPE_CHECKING + from .base import ComponentBuilder +if TYPE_CHECKING: + from collections.abc import Sequence + + from python_discovery import PythonInfo + + from virtualenv.config.cli.parser import VirtualEnvConfigParser, VirtualEnvOptions + from virtualenv.seed.seeder import Seeder + class SeederSelector(ComponentBuilder): - def __init__(self, interpreter, parser) -> None: + def __init__(self, interpreter: PythonInfo, parser: VirtualEnvConfigParser) -> None: possible = self.options("virtualenv.seed") super().__init__(interpreter, parser, "seeder", possible) - def add_selector_arg_parse(self, name, choices): + def add_selector_arg_parse(self, name: str, choices: Sequence[str]) -> None: self.parser.add_argument( f"--{name}", choices=choices, @@ -25,14 +35,18 @@ def add_selector_arg_parse(self, name, choices): ) @staticmethod - def _get_default(): + def _get_default() -> str: return "app-data" - def handle_selected_arg_parse(self, options): + def handle_selected_arg_parse(self, options: VirtualEnvOptions) -> str: return super().handle_selected_arg_parse(options) - def create(self, options): - return self._impl_class(options) + def create(self, options: VirtualEnvOptions) -> Seeder: + assert self._impl_class is not None # ruff:ignore[assert] # Set by handle_selected_arg_parse + seeder = self._impl_class(options) + if seeder.enabled and (reason := seeder.cannot_seed(self.interpreter)) is not None: + raise RuntimeError(reason) + return seeder __all__ = [ diff --git a/src/virtualenv/run/session.py b/src/virtualenv/run/session.py index def795328..9006e8adb 100644 --- a/src/virtualenv/run/session.py +++ b/src/virtualenv/run/session.py @@ -2,6 +2,23 @@ import json import logging +import sys +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from types import TracebackType + + from python_discovery import PythonInfo + + from virtualenv.activation.activator import Activator + from virtualenv.app_data.base import AppData + from virtualenv.create.creator import Creator + from virtualenv.seed.seeder import Seeder + +if sys.version_info >= (3, 11): + from typing import Self +else: + from typing_extensions import Self LOGGER = logging.getLogger(__name__) @@ -9,7 +26,15 @@ class Session: """Represents a virtual environment creation session.""" - def __init__(self, verbosity, app_data, interpreter, creator, seeder, activators) -> None: # noqa: PLR0913 + def __init__( # ruff:ignore[too-many-arguments] + self, + verbosity: int, + app_data: AppData, + interpreter: PythonInfo, + creator: Creator, + seeder: Seeder, + activators: list[Activator], + ) -> None: self._verbosity = verbosity self._app_data = app_data self._interpreter = interpreter @@ -18,58 +43,63 @@ def __init__(self, verbosity, app_data, interpreter, creator, seeder, activators self._activators = activators @property - def verbosity(self): + def verbosity(self) -> int: """The verbosity of the run.""" return self._verbosity @property - def interpreter(self): + def interpreter(self) -> PythonInfo: """Create a virtual environment based on this reference interpreter.""" return self._interpreter @property - def creator(self): + def creator(self) -> Creator: """The creator used to build the virtual environment (must be compatible with the interpreter).""" return self._creator @property - def seeder(self): + def seeder(self) -> Seeder: """The mechanism used to provide the seed packages (pip, setuptools, wheel).""" return self._seeder @property - def activators(self): + def activators(self) -> list[Activator]: """Activators used to generate activations scripts.""" return self._activators - def run(self): + def run(self) -> None: self._create() self._seed() self._activate() self.creator.pyenv_cfg.write() - def _create(self): + def _create(self) -> None: LOGGER.info("create virtual environment via %s", self.creator) self.creator.run() LOGGER.debug(_DEBUG_MARKER) LOGGER.debug("%s", _Debug(self.creator)) - def _seed(self): + def _seed(self) -> None: if self.seeder is not None and self.seeder.enabled: LOGGER.info("add seed packages via %s", self.seeder) self.seeder.run(self.creator) - def _activate(self): + def _activate(self) -> None: if self.activators: active = ", ".join(type(i).__name__.replace("Activator", "") for i in self.activators) LOGGER.info("add activators for %s", active) for activator in self.activators: activator.generate(self.creator) - def __enter__(self): + def __enter__(self) -> Self: return self - def __exit__(self, exc_type, exc_val, exc_tb): + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> None: self._app_data.close() @@ -79,7 +109,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): class _Debug: """lazily populate debug.""" - def __init__(self, creator) -> None: + def __init__(self, creator: Creator) -> None: self.creator = creator def __repr__(self) -> str: diff --git a/src/virtualenv/seed/embed/base_embed.py b/src/virtualenv/seed/embed/base_embed.py index 72fc5a34a..65c2f4ebd 100644 --- a/src/virtualenv/seed/embed/base_embed.py +++ b/src/virtualenv/seed/embed/base_embed.py @@ -4,16 +4,26 @@ from abc import ABC from argparse import SUPPRESS from pathlib import Path +from typing import TYPE_CHECKING from virtualenv.seed.seeder import Seeder from virtualenv.seed.wheels import Version +from virtualenv.seed.wheels.embed import MIN, OLDEST_SUPPORTED + +if TYPE_CHECKING: + from argparse import ArgumentParser + + from python_discovery import PythonInfo + + from virtualenv.app_data.base import AppData + from virtualenv.config.cli.parser import VirtualEnvOptions LOGGER = logging.getLogger(__name__) PERIODIC_UPDATE_ON_BY_DEFAULT = True class BaseEmbed(Seeder, ABC): - def __init__(self, options) -> None: + def __init__(self, options: VirtualEnvOptions) -> None: super().__init__(options, enabled=options.no_seed is False) self.download = options.download @@ -22,31 +32,27 @@ def __init__(self, options) -> None: self.pip_version = options.pip self.setuptools_version = options.setuptools - # wheel version needs special handling - # on Python > 3.8, the default is None (as in not used) - # so we can differentiate between explicit and implicit none + # virtualenv no longer bundles wheel; the parsed default stays None (unused) so the + # warning below fires only when you pass --wheel or --no-wheel self.wheel_version = options.wheel or "none" self.no_pip = options.no_pip self.no_setuptools = options.no_setuptools - self.no_wheel = options.no_wheel self.app_data = options.app_data self.periodic_update = not options.no_periodic_update - if options.py_version[:2] >= (3, 9): - if options.wheel is not None or options.no_wheel: - LOGGER.warning( - "The --no-wheel and --wheel options are deprecated. " - "They have no effect for Python > 3.8 as wheel is no longer " - "bundled in virtualenv.", - ) - self.no_wheel = True + if options.wheel is not None or options.no_wheel: + LOGGER.warning( + "DEPRECATION: the --wheel and --no-wheel options do nothing; virtualenv no longer bundles wheel. " + "They will be removed in a release after 2026-12. Stop passing them.", + ) + self.no_wheel = True if not self.distribution_to_versions(): self.enabled = False @classmethod - def distributions(cls) -> dict[str, Version]: + def distributions(cls) -> dict[str, str]: return { "pip": Version.bundle, "setuptools": Version.bundle, @@ -61,7 +67,29 @@ def distribution_to_versions(self) -> dict[str, str]: } @classmethod - def add_parser_arguments(cls, parser, interpreter, app_data): # noqa: ARG003 + def cannot_seed(cls, interpreter: PythonInfo) -> str | None: + """Explain why the bundled wheels cannot seed the target Python version. + + The embedded pip/setuptools stopped shipping for Pythons below :data:`OLDEST_SUPPORTED`, so seeding one would + install an incompatible wheel. + + :param interpreter: the interpreter to be seeded + + :returns: ``None`` when the bundled wheels still support the target, otherwise a message naming the target and + the remedies + + """ + if interpreter.version_info[:2] >= OLDEST_SUPPORTED: + return None + target = f"{interpreter.version_info.major}.{interpreter.version_info.minor}" + return ( + f"the bundled seeder no longer ships pip/setuptools for Python {target}; the oldest supported target is " + f"Python {MIN} - pass --no-seed for an empty environment, use a seeder that provides Python {target} " + f"wheels, or install an older virtualenv release" + ) + + @classmethod + def add_parser_arguments(cls, parser: ArgumentParser, interpreter: PythonInfo, app_data: AppData) -> None: # ruff:ignore[unused-class-method-argument] group = parser.add_mutually_exclusive_group() group.add_argument( "--no-download", @@ -88,10 +116,10 @@ def add_parser_arguments(cls, parser, interpreter, app_data): # noqa: ARG003 ) for distribution, default in cls.distributions().items(): help_ = f"version of {distribution} to install as seed: embed, bundle, none or exact version" - if interpreter.version_info[:2] >= (3, 12) and distribution in {"wheel", "setuptools"}: - default = "none" # noqa: PLW2901 - if interpreter.version_info[:2] >= (3, 9) and distribution == "wheel": - default = None # noqa: PLW2901 + if interpreter.version_info[:2] >= (3, 12) and distribution == "setuptools": + default = "none" # ruff:ignore[redefined-loop-name] + if distribution == "wheel": + default = None # ruff:ignore[redefined-loop-name] help_ = SUPPRESS parser.add_argument( f"--{distribution}", @@ -102,7 +130,7 @@ def add_parser_arguments(cls, parser, interpreter, app_data): # noqa: ARG003 ) for distribution in cls.distributions(): help_ = f"do not install {distribution}" - if interpreter.version_info[:2] >= (3, 9) and distribution == "wheel": + if distribution == "wheel": help_ = SUPPRESS parser.add_argument( f"--no-{distribution}", diff --git a/src/virtualenv/seed/embed/pip_invoke.py b/src/virtualenv/seed/embed/pip_invoke.py index b733c5148..fb442291c 100644 --- a/src/virtualenv/seed/embed/pip_invoke.py +++ b/src/virtualenv/seed/embed/pip_invoke.py @@ -3,19 +3,27 @@ import logging from contextlib import contextmanager from subprocess import Popen +from typing import TYPE_CHECKING -from virtualenv.discovery.cached_py_info import LogCmd from virtualenv.seed.embed.base_embed import BaseEmbed from virtualenv.seed.wheels import Version, get_wheel, pip_wheel_env_run +from virtualenv.util.subprocess import LogCmd + +if TYPE_CHECKING: + from collections.abc import Generator + from pathlib import Path + + from virtualenv.config.cli.parser import VirtualEnvOptions + from virtualenv.create.creator import Creator LOGGER = logging.getLogger(__name__) class PipInvoke(BaseEmbed): - def __init__(self, options) -> None: + def __init__(self, options: VirtualEnvOptions) -> None: super().__init__(options) - def run(self, creator): + def run(self, creator: Creator) -> None: if not self.enabled: return for_py_version = creator.interpreter.version_release_str @@ -24,7 +32,7 @@ def run(self, creator): self._execute(cmd, env) @staticmethod - def _execute(cmd, env): + def _execute(cmd: list[str], env: dict[str, str]) -> Popen[bytes]: LOGGER.debug("pip seed by running: %s", LogCmd(cmd, env)) process = Popen(cmd, env=env) process.communicate() @@ -34,8 +42,18 @@ def _execute(cmd, env): return process @contextmanager - def get_pip_install_cmd(self, exe, for_py_version): - cmd = [str(exe), "-m", "pip", "-q", "install", "--only-binary", ":all:", "--disable-pip-version-check"] + def get_pip_install_cmd(self, exe: Path, for_py_version: str) -> Generator[list[str], None, None]: + cmd = [ + str(exe), + "-m", + "pip", + "-q", + "install", + "--only-binary", + ":all:", + "--disable-pip-version-check", + "--ignore-installed", + ] if not self.download: cmd.append("--no-index") folders = set() diff --git a/src/virtualenv/seed/embed/via_app_data/pip_install/base.py b/src/virtualenv/seed/embed/via_app_data/pip_install/base.py index d5e87ad93..39d5a3b90 100644 --- a/src/virtualenv/seed/embed/via_app_data/pip_install/base.py +++ b/src/virtualenv/seed/embed/via_app_data/pip_install/base.py @@ -1,7 +1,9 @@ from __future__ import annotations import logging +import ntpath import os +import posixpath import re import zipfile from abc import ABC, abstractmethod @@ -9,16 +11,38 @@ from itertools import chain from pathlib import Path from tempfile import mkdtemp +from typing import TYPE_CHECKING from distlib.scripts import ScriptMaker, enquote_executable from virtualenv.util.path import safe_delete +if TYPE_CHECKING: + from virtualenv.create.creator import Creator + LOGGER = logging.getLogger(__name__) +def _safe_extract_zip(zip_ref: zipfile.ZipFile, target_dir: Path) -> None: + # Guard against zip slip: a wheel is a zip and a tampered entry name (absolute path or one containing ``..``) + # could escape ``target_dir``. + base = target_dir.resolve() + for info in zip_ref.infolist(): + name = info.filename + if name.startswith(("/", "\\")) or ntpath.isabs(name) or posixpath.isabs(name): + msg = f"refusing to extract absolute path entry from wheel: {name!r}" + raise RuntimeError(msg) + candidate = (base / name).resolve() + try: + candidate.relative_to(base) + except ValueError as exc: + msg = f"refusing to extract entry escaping target directory: {name!r}" + raise RuntimeError(msg) from exc + zip_ref.extractall(str(target_dir)) + + class PipInstall(ABC): - def __init__(self, wheel, creator, image_folder) -> None: + def __init__(self, wheel: Path, creator: Creator, image_folder: Path) -> None: self._wheel = wheel self._creator = creator self._image_dir = image_folder @@ -27,10 +51,10 @@ def __init__(self, wheel, creator, image_folder) -> None: self._console_entry_points = None @abstractmethod - def _sync(self, src, dst): + def _sync(self, src: Path, dst: Path) -> None: raise NotImplementedError - def install(self, version_info): + def install(self, version_info: tuple[int, ...]) -> None: self._extracted = True self._uninstall_previous_version() # sync image @@ -40,46 +64,54 @@ def install(self, version_info): # generate console executables consoles = set() script_dir = self._creator.script_dir - for name, module in self._console_scripts.items(): + for name, module in self._console_scripts.items(): # ty: ignore[unresolved-attribute] consoles.update(self._create_console_entry_point(name, module, script_dir, version_info)) LOGGER.debug("generated console scripts %s", " ".join(i.name for i in consoles)) - def build_image(self): + def build_image(self) -> None: + """Extract the seed wheel into the image directory and fix up its RECORD file. + + Each archive entry is validated before extraction so a tampered wheel cannot escape the image directory via an + absolute path or ``..`` traversal. + + :raises RuntimeError: if the wheel contains an entry that would land outside the image directory. + + """ # 1. first extract the wheel LOGGER.debug("build install image for %s to %s", self._wheel.name, self._image_dir) with zipfile.ZipFile(str(self._wheel)) as zip_ref: self._shorten_path_if_needed(zip_ref) - zip_ref.extractall(str(self._image_dir)) + _safe_extract_zip(zip_ref, self._image_dir) self._extracted = True # 2. now add additional files not present in the distribution new_files = self._generate_new_files() # 3. finally fix the records file self._fix_records(new_files) - def _shorten_path_if_needed(self, zip_ref): + def _shorten_path_if_needed(self, zip_ref: zipfile.ZipFile) -> None: if os.name == "nt": to_folder = str(self._image_dir) # https://docs.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation zip_max_len = max(len(i) for i in zip_ref.namelist()) path_len = zip_max_len + len(to_folder) - if path_len > 260: # noqa: PLR2004 + if path_len > 260: # ruff:ignore[magic-value-comparison] self._image_dir.mkdir(exist_ok=True) # to get a short path must exist - from virtualenv.util.path import get_short_path_name # noqa: PLC0415 + from virtualenv.util.path import get_short_path_name # ruff:ignore[import-outside-top-level] to_folder = get_short_path_name(to_folder) self._image_dir = Path(to_folder) - def _records_text(self, files): + def _records_text(self, files: set[Path] | list[Path]) -> str: return "\n".join(f"{os.path.relpath(str(rec), str(self._image_dir))},," for rec in files) - def _generate_new_files(self): + def _generate_new_files(self) -> set[Path]: new_files = set() - installer = self._dist_info / "INSTALLER" + installer = self._dist_info / "INSTALLER" # ty: ignore[unsupported-operator] installer.write_text("pip\n", encoding="utf-8") new_files.add(installer) # inject a no-op root element, as workaround for bug in https://github.com/pypa/pip/issues/7226 - marker = self._image_dir / f"{self._dist_info.stem}.virtualenv" + marker = self._image_dir / f"{self._dist_info.stem}.virtualenv" # ty: ignore[unresolved-attribute] marker.write_text("", encoding="utf-8") new_files.add(marker) folder = mkdtemp() @@ -87,17 +119,17 @@ def _generate_new_files(self): to_folder = Path(folder) rel = os.path.relpath(str(self._creator.script_dir), str(self._creator.purelib)) version_info = self._creator.interpreter.version_info - for name, module in self._console_scripts.items(): + for name, module in self._console_scripts.items(): # ty: ignore[unresolved-attribute] new_files.update( Path(os.path.normpath(str(self._image_dir / rel / i.name))) - for i in self._create_console_entry_point(name, module, to_folder, version_info) + for i in self._create_console_entry_point(name, module, to_folder, version_info) # ty: ignore[invalid-argument-type] ) finally: - safe_delete(folder) + safe_delete(folder) # ty: ignore[invalid-argument-type] return new_files @property - def _dist_info(self): + def _dist_info(self) -> Path | None: if self._extracted is False: return None # pragma: no cover if self.__dist_info is None: @@ -113,16 +145,16 @@ def _dist_info(self): return self.__dist_info @abstractmethod - def _fix_records(self, extra_record_data): + def _fix_records(self, extra_record_data: set[Path]) -> None: raise NotImplementedError @property - def _console_scripts(self): + def _console_scripts(self) -> dict[str, str] | None: if self._extracted is False: return None # pragma: no cover if self._console_entry_points is None: self._console_entry_points = {} - entry_points = self._dist_info / "entry_points.txt" + entry_points = self._dist_info / "entry_points.txt" # ty: ignore[unsupported-operator] if entry_points.exists(): parser = ConfigParser() with entry_points.open(encoding="utf-8") as file_handler: @@ -130,11 +162,13 @@ def _console_scripts(self): if "console_scripts" in parser.sections(): for name, value in parser.items("console_scripts"): match = re.match(r"(.*?)-?\d\.?\d*", name) - our_name = match.groups(1)[0] if match else name + our_name = match.group(1) if match else name self._console_entry_points[our_name] = value return self._console_entry_points - def _create_console_entry_point(self, name, value, to_folder, version_info): + def _create_console_entry_point( + self, name: str, value: str, to_folder: Path, version_info: tuple[int, ...] + ) -> list[Path]: result = [] maker = ScriptMakerCustom(to_folder, version_info, self._creator.exe, name) specification = f"{name} = {value}" @@ -142,8 +176,8 @@ def _create_console_entry_point(self, name, value, to_folder, version_info): result.extend(Path(i) for i in new_files) return result - def _uninstall_previous_version(self): - dist_name = self._dist_info.stem.split("-")[0] + def _uninstall_previous_version(self) -> None: + dist_name = self._dist_info.stem.split("-")[0] # ty: ignore[unresolved-attribute] in_folders = chain.from_iterable([i.iterdir() for i in (self._creator.purelib, self._creator.platlib)]) paths = (p for p in in_folders if p.stem.split("-")[0] == dist_name and p.suffix == ".dist-info" and p.is_dir()) existing_dist = next(paths, None) @@ -151,7 +185,7 @@ def _uninstall_previous_version(self): self._uninstall_dist(existing_dist) @staticmethod - def _uninstall_dist(dist): + def _uninstall_dist(dist: Path) -> None: dist_base = dist.parent LOGGER.debug("uninstall existing distribution %s from %s", dist.stem, dist_base) @@ -178,25 +212,27 @@ def _uninstall_dist(dist): else: path.unlink() - def clear(self): + def clear(self) -> None: if self._image_dir.exists(): safe_delete(self._image_dir) - def has_image(self): + def has_image(self) -> bool: return self._image_dir.exists() and any(self._image_dir.iterdir()) class ScriptMakerCustom(ScriptMaker): - def __init__(self, target_dir, version_info, executable, name) -> None: + def __init__(self, target_dir: Path, version_info: tuple[int, ...], executable: Path, name: str) -> None: super().__init__(None, str(target_dir)) self.clobber = True # overwrite self.set_mode = True # ensure they are executable self.executable = enquote_executable(str(executable)) - self.version_info = version_info.major, version_info.minor + self.version_info = version_info.major, version_info.minor # ty: ignore[unresolved-attribute] self.variants = {"", "X", "X.Y"} self._name = name - def _write_script(self, names, shebang, script_bytes, filenames, ext): + def _write_script( + self, names: set[str], shebang: bytes, script_bytes: bytes, filenames: list[str], ext: str + ) -> None: names.add(f"{self._name}{self.version_info[0]}.{self.version_info[1]}") super()._write_script(names, shebang, script_bytes, filenames, ext) diff --git a/src/virtualenv/seed/embed/via_app_data/pip_install/copy.py b/src/virtualenv/seed/embed/via_app_data/pip_install/copy.py index b5f01aa91..e218bf0b1 100644 --- a/src/virtualenv/seed/embed/via_app_data/pip_install/copy.py +++ b/src/virtualenv/seed/embed/via_app_data/pip_install/copy.py @@ -2,23 +2,27 @@ import os from pathlib import Path +from typing import TYPE_CHECKING from virtualenv.util.path import copy from .base import PipInstall +if TYPE_CHECKING: + from collections.abc import Generator + class CopyPipInstall(PipInstall): - def _sync(self, src, dst): + def _sync(self, src: Path, dst: Path) -> None: copy(src, dst) - def _generate_new_files(self): + def _generate_new_files(self) -> set[Path]: # create the pyc files new_files = super()._generate_new_files() new_files.update(self._cache_files()) return new_files - def _cache_files(self): + def _cache_files(self) -> Generator[Path, None, None]: version = self._creator.interpreter.version_info py_c_ext = f".{self._creator.interpreter.implementation.lower()}-{version.major}{version.minor}.pyc" for root, dirs, files in os.walk(str(self._image_dir), topdown=True): @@ -29,9 +33,9 @@ def _cache_files(self): for name in dirs: yield root_path / name / "__pycache__" - def _fix_records(self, new_files): - extra_record_data_str = self._records_text(new_files) - with (self._dist_info / "RECORD").open("ab") as file_handler: + def _fix_records(self, extra_record_data: set[Path]) -> None: + extra_record_data_str = self._records_text(extra_record_data) + with (self._dist_info / "RECORD").open("ab") as file_handler: # ty: ignore[unsupported-operator] file_handler.write(extra_record_data_str.encode("utf-8")) diff --git a/src/virtualenv/seed/embed/via_app_data/pip_install/symlink.py b/src/virtualenv/seed/embed/via_app_data/pip_install/symlink.py index 7eb9f5f47..855285673 100644 --- a/src/virtualenv/seed/embed/via_app_data/pip_install/symlink.py +++ b/src/virtualenv/seed/embed/via_app_data/pip_install/symlink.py @@ -3,17 +3,21 @@ import os from stat import S_IREAD, S_IRGRP, S_IROTH from subprocess import PIPE, Popen +from typing import TYPE_CHECKING from virtualenv.util.path import safe_delete, set_tree from .base import PipInstall +if TYPE_CHECKING: + from pathlib import Path + class SymlinkPipInstall(PipInstall): - def _sync(self, src, dst): + def _sync(self, src: Path, dst: Path) -> None: os.symlink(str(src), str(dst)) - def _generate_new_files(self): + def _generate_new_files(self) -> set[Path]: # create the pyc files, as the build image will be R/O cmd = [str(self._creator.exe), "-m", "compileall", str(self._image_dir)] process = Popen(cmd, stdout=PIPE, stderr=PIPE) @@ -37,17 +41,17 @@ def _generate_new_files(self): new_files.add(file) return new_files - def _fix_records(self, new_files): - new_files.update(i for i in self._image_dir.iterdir()) - extra_record_data_str = self._records_text(sorted(new_files, key=str)) - (self._dist_info / "RECORD").write_text(extra_record_data_str, encoding="utf-8") + def _fix_records(self, extra_record_data: set[Path]) -> None: + extra_record_data.update(i for i in self._image_dir.iterdir()) + extra_record_data_str = self._records_text(sorted(extra_record_data, key=str)) # ty: ignore[invalid-argument-type] + (self._dist_info / "RECORD").write_text(extra_record_data_str, encoding="utf-8") # ty: ignore[unsupported-operator] - def build_image(self): + def build_image(self) -> None: super().build_image() # protect the image by making it read only set_tree(self._image_dir, S_IREAD | S_IRGRP | S_IROTH) - def clear(self): + def clear(self) -> None: if self._image_dir.exists(): safe_delete(self._image_dir) super().clear() diff --git a/src/virtualenv/seed/embed/via_app_data/via_app_data.py b/src/virtualenv/seed/embed/via_app_data/via_app_data.py index a2e9630c6..48f2767c3 100644 --- a/src/virtualenv/seed/embed/via_app_data/via_app_data.py +++ b/src/virtualenv/seed/embed/via_app_data/via_app_data.py @@ -9,6 +9,7 @@ from pathlib import Path from subprocess import CalledProcessError from threading import Lock, Thread +from typing import TYPE_CHECKING from virtualenv.info import fs_supports_symlink from virtualenv.seed.embed.base_embed import BaseEmbed @@ -17,16 +18,30 @@ from .pip_install.copy import CopyPipInstall from .pip_install.symlink import SymlinkPipInstall +if TYPE_CHECKING: + from argparse import ArgumentParser + from collections.abc import Generator + + from python_discovery import PythonInfo + + from virtualenv.app_data.base import AppData + from virtualenv.config.cli.parser import VirtualEnvOptions + from virtualenv.create.creator import Creator + from virtualenv.seed.wheels.util import Wheel + from virtualenv.util.lock import PathLockBase + + from .pip_install.base import PipInstall + LOGGER = logging.getLogger(__name__) class FromAppData(BaseEmbed): - def __init__(self, options) -> None: + def __init__(self, options: VirtualEnvOptions) -> None: super().__init__(options) self.symlinks = options.symlink_app_data @classmethod - def add_parser_arguments(cls, parser, interpreter, app_data): + def add_parser_arguments(cls, parser: ArgumentParser, interpreter: PythonInfo, app_data: AppData) -> None: super().add_parser_arguments(parser, interpreter, app_data) can_symlink = app_data.transient is False and fs_supports_symlink() sym = "" if can_symlink else "not supported - " @@ -38,7 +53,7 @@ def add_parser_arguments(cls, parser, interpreter, app_data): default=False, ) - def run(self, creator): + def run(self, creator: Creator) -> None: if not self.enabled: return with self._get_seed_wheels(creator) as name_to_whl: @@ -46,18 +61,15 @@ def run(self, creator): installer_class = self.installer_class(pip_version) exceptions = {} - def _install(name, wheel): + def _install(name: str, wheel: Wheel) -> None: + LOGGER.debug("install %s from wheel %s via %s", name, wheel, installer_class.__name__) try: - LOGGER.debug("install %s from wheel %s via %s", name, wheel, installer_class.__name__) key = Path(installer_class.__name__) / wheel.path.stem wheel_img = self.app_data.wheel_image(creator.interpreter.version_release_str, key) installer = installer_class(wheel.path, creator, wheel_img) - parent = self.app_data.lock / wheel_img.parent - with parent.non_reentrant_lock_for_key(wheel_img.name): - if not installer.has_image(): - installer.build_image() - installer.install(creator.interpreter.version_info) - except Exception: # noqa: BLE001 + _build_wheel_image(self.app_data.lock / wheel_img.parent, wheel_img.name, installer) + installer.install(creator.interpreter.version_info) # ty: ignore[invalid-argument-type] + except Exception: # ruff:ignore[blind-except] exceptions[name] = sys.exc_info() threads = [Thread(target=_install, args=(n, w)) for n, w in name_to_whl.items()] @@ -73,10 +85,10 @@ def _install(name, wheel): raise RuntimeError("\n".join(messages)) @contextmanager - def _get_seed_wheels(self, creator): # noqa: C901 + def _get_seed_wheels(self, creator: Creator) -> Generator[dict[str, Wheel], None, None]: # ruff:ignore[complex-structure] name_to_whl, lock, fail = {}, Lock(), {} - def _get(distribution, version): + def _get(distribution: str, version: str | None) -> None: for_py_version = creator.interpreter.version_release_str failure, result = None, None # fallback to download in case the exact version is not available @@ -130,7 +142,7 @@ def _get(distribution, version): raise RuntimeError(msg) yield name_to_whl - def installer_class(self, pip_version_tuple): + def installer_class(self, pip_version_tuple: tuple[int, ...] | None) -> type[PipInstall]: if self.symlinks and pip_version_tuple and pip_version_tuple >= (19, 3): # symlink support requires pip 19.3+ return SymlinkPipInstall return CopyPipInstall @@ -141,6 +153,12 @@ def __repr__(self) -> str: return f"{base[:-1]}{msg}{base[-1]}" +def _build_wheel_image(parent: PathLockBase, name: str, installer: PipInstall) -> None: + with parent.non_reentrant_lock_for_key(name): + if not installer.has_image(): + installer.build_image() + + __all__ = [ "FromAppData", ] diff --git a/src/virtualenv/seed/seeder.py b/src/virtualenv/seed/seeder.py index 58fd8f416..4bb03ae1f 100644 --- a/src/virtualenv/seed/seeder.py +++ b/src/virtualenv/seed/seeder.py @@ -1,39 +1,61 @@ from __future__ import annotations from abc import ABC, abstractmethod +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from argparse import ArgumentParser + + from python_discovery import PythonInfo + + from virtualenv.app_data.base import AppData + from virtualenv.config.cli.parser import VirtualEnvOptions + from virtualenv.create.creator import Creator class Seeder(ABC): """A seeder will install some seed packages into a virtual environment.""" - def __init__(self, options, enabled) -> None: - """ - Create. + def __init__(self, options: VirtualEnvOptions, enabled: bool) -> None: + """Create. :param options: the parsed options as defined within :meth:`add_parser_arguments` :param enabled: a flag weather the seeder is enabled or not + """ self.enabled = enabled self.env = options.env @classmethod - def add_parser_arguments(cls, parser, interpreter, app_data): + def cannot_seed(cls, interpreter: PythonInfo) -> str | None: # ruff:ignore[unused-class-method-argument] + """Explain why this seeder cannot install seed packages for the given interpreter. + + :param interpreter: the interpreter the environment is based on + + :returns: ``None`` when the seeder supports the interpreter, otherwise a message describing why it cannot; + selection rejects a seeder that returns a message and surfaces it to the user + """ - Add CLI arguments for this seed mechanisms. + return None + + @classmethod + def add_parser_arguments(cls, parser: ArgumentParser, interpreter: PythonInfo, app_data: AppData) -> None: + """Add CLI arguments for this seed mechanisms. :param parser: the CLI parser :param app_data: the CLI parser :param interpreter: the interpreter this virtual environment is based of + """ raise NotImplementedError @abstractmethod - def run(self, creator): - """ - Perform the seed operation. + def run(self, creator: Creator) -> None: + """Perform the seed operation. + + :param creator: the creator (based of :class:`virtualenv.create.creator.Creator`) we used to create this virtual + environment - :param creator: the creator (based of :class:`virtualenv.create.creator.Creator`) we used to create this \ - virtual environment """ raise NotImplementedError diff --git a/src/virtualenv/seed/wheels/acquire.py b/src/virtualenv/seed/wheels/acquire.py index eb2fb5b45..bd42796df 100644 --- a/src/virtualenv/seed/wheels/acquire.py +++ b/src/virtualenv/seed/wheels/acquire.py @@ -3,28 +3,76 @@ from __future__ import annotations import logging +import re import sys -from operator import eq, lt from pathlib import Path from subprocess import PIPE, CalledProcessError, Popen +from typing import TYPE_CHECKING, Final + +from packaging.specifiers import Specifier from .bundle import from_bundle -from .periodic_update import add_wheel_to_update_log +from .periodic_update import UnverifiedWheelError, add_wheel_to_update_log, verify_wheel_digest from .util import Version, Wheel, discover_wheels -LOGGER = logging.getLogger(__name__) +if TYPE_CHECKING: + from virtualenv.app_data.base import AppData +LOGGER = logging.getLogger(__name__) -def get_wheel( # noqa: PLR0913 - distribution, - version, - for_py_version, - search_dirs, - download, - app_data, - do_periodic_update, - env, -): +# A private index can legitimately serve a rebuilt wheel under the same distribution, version, and +# filename PyPI also uses - an internal mirror re-signing or re-packaging a release for policy reasons, +# for example. Verifying that file against PyPI's own digest would then reject a wheel that came from +# exactly the source it was configured to come from, not a compromised one. This does not attempt to +# find every way pip can be pointed elsewhere (a pip.conf file has the same effect and is invisible +# here), only the common, explicit one, so treat this as a conservative signal rather than proof either +# way: it exists to avoid false rejections, not to guarantee verification runs whenever it safely could. +_CUSTOM_INDEX_ENV_VARS: Final[tuple[str, ...]] = ("PIP_INDEX_URL", "PIP_EXTRA_INDEX_URL", "PIP_INDEX") + + +def _uses_default_index(env: dict[str, str]) -> bool: + return not any(env.get(var) for var in _CUSTOM_INDEX_ENV_VARS) + + +# PEP 503 normalized distribution name. Anything outside this character set on the way to ``pip download`` means +# somebody is smuggling pip options or extras, so reject it before we build the command line. +_DISTRIBUTION_RE = re.compile( + r""" + ^ + (?P + [A-Za-z0-9] # must start with an alnum + (?:[A-Za-z0-9._-]* # inner chars: alnum plus . _ - + [A-Za-z0-9])? # must also end with an alnum (unless length is 1) + ) + $ + """, + re.VERBOSE, +) + +# Version specifier that matches what ``Version.as_version_spec`` emits: either empty, ``==`` or ``<`` where +# ```` is a subset of PEP 440 public versions. Kept deliberately strict so a crafted version cannot inject pip +# flags. +_VERSION_SPEC_RE = re.compile( + r""" + ^ + (?P==|<) # only the operators Version.as_version_spec can emit + (?P[A-Za-z0-9._+!-]+) # PEP 440 public-version character set, no whitespace + $ + """, + re.VERBOSE, +) + + +def get_wheel( # ruff:ignore[too-many-arguments] + distribution: str, + version: str | None, + for_py_version: str, + search_dirs: list[Path], + download: bool, + app_data: AppData, + do_periodic_update: bool, + env: dict[str, str], +) -> Wheel | None: """Get a wheel with the given distribution-version-for_py_version trio, by using the extra search dir + download.""" # not all wheels are compatible with all python versions, so we need to py version qualify it wheel = None @@ -35,22 +83,60 @@ def get_wheel( # noqa: PLR0913 if download and wheel is None and version != Version.embed: # 2. download from the internet - wheel = download_wheel( - distribution=distribution, - version_spec=Version.as_version_spec(version), - for_py_version=for_py_version, - search_dirs=search_dirs, - app_data=app_data, - to_folder=app_data.house, - env=env, - ) + try: + wheel = download_wheel( + distribution=distribution, + version_spec=Version.as_version_spec(version), + for_py_version=for_py_version, + search_dirs=search_dirs, + app_data=app_data, + to_folder=app_data.house, + env=env, + ) + except UnverifiedWheelError as exception: + # only the bundle request has a local wheel that satisfies it; a pinned version must fail + if version != Version.bundle: + raise + LOGGER.warning("%s, seeding the bundled %s instead", exception, distribution) + return from_bundle(distribution, version, for_py_version, search_dirs, app_data, do_periodic_update, env) if wheel is not None and app_data.can_update: add_wheel_to_update_log(wheel, for_py_version, app_data) return wheel -def download_wheel(distribution, version_spec, for_py_version, search_dirs, app_data, to_folder, env): # noqa: PLR0913 +def download_wheel( # ruff:ignore[too-many-arguments] + distribution: str, + version_spec: str | None, + for_py_version: str, + search_dirs: list[Path], + app_data: AppData, + to_folder: Path, + env: dict[str, str], +) -> Wheel: + """Invoke ``pip download`` in a subprocess to fetch a seed wheel. + + :param distribution: PEP 503 normalized project name; rejected if it contains anything other than + ``[A-Za-z0-9._-]``. + :param version_spec: optional version specifier of the form ``==`` or ``<`` as emitted by + :func:`Version.as_version_spec`, or ``None``/empty for the latest compatible release. + :param for_py_version: major.minor Python version to pass through to ``pip --python-version``. + :param search_dirs: additional directories to treat as a local wheel index when bootstrapping pip. + :param app_data: application data store used to locate the embedded pip wheel. + :param to_folder: directory the downloaded wheel is written into. + :param env: environment mapping passed through to the subprocess. + + :returns: the downloaded :class:`Wheel`. + + :raises ValueError: if ``distribution`` or ``version_spec`` fail the strict allow-list check. + :raises CalledProcessError: if ``pip download`` exits with a non-zero status. + :raises RuntimeError: if no downloaded wheel can be identified from ``pip``'s output or the search directory, or if + the caller has not configured a custom index and PyPI's sha256 for the downloaded filename cannot be obtained or + does not match, see :func:`virtualenv.seed.wheels.periodic_update.verify_wheel_digest`. + + """ + _check_distribution(distribution) + _check_version_spec(version_spec) to_download = f"{distribution}{version_spec or ''}" LOGGER.debug("download wheel %s %s to %s", to_download, for_py_version, to_folder) cmd = [ @@ -69,6 +155,7 @@ def download_wheel(distribution, version_spec, for_py_version, search_dirs, app_ str(to_folder), to_download, ] + verify_against_pypi = _uses_default_index(env) # pip has no interface in python - must be a new sub-process env = pip_wheel_env_run(search_dirs, app_data, env) process = Popen(cmd, env=env, stdout=PIPE, stderr=PIPE, universal_newlines=True, encoding="utf-8") @@ -77,11 +164,18 @@ def download_wheel(distribution, version_spec, for_py_version, search_dirs, app_ kwargs = {"output": out, "stderr": err} raise CalledProcessError(process.returncode, cmd, **kwargs) result = _find_downloaded_wheel(distribution, version_spec, for_py_version, to_folder, out) + if result is None: + msg = f"could not find downloaded wheel for {to_download}" + raise RuntimeError(msg) LOGGER.debug("downloaded wheel %s", result.name) + if verify_against_pypi: + verify_wheel_digest(result) return result -def _find_downloaded_wheel(distribution, version_spec, for_py_version, to_folder, out): +def _find_downloaded_wheel( + distribution: str, version_spec: str | None, for_py_version: str, to_folder: Path, out: str +) -> Wheel | None: for line in out.splitlines(): stripped_line = line.lstrip() for marker in ("Saved ", "File was already downloaded "): @@ -91,23 +185,19 @@ def _find_downloaded_wheel(distribution, version_spec, for_py_version, to_folder return find_compatible_in_house(distribution, version_spec, for_py_version, to_folder) -def find_compatible_in_house(distribution, version_spec, for_py_version, in_folder): - wheels = discover_wheels(in_folder, distribution, None, for_py_version) - start, end = 0, len(wheels) - if version_spec is not None and version_spec: - if version_spec.startswith("<"): - from_pos, op = 1, lt - elif version_spec.startswith("=="): - from_pos, op = 2, eq - else: +def find_compatible_in_house( + distribution: str, version_spec: str | None, for_py_version: str, in_folder: Path +) -> Wheel | None: + wheels: Final[list[Wheel]] = discover_wheels(in_folder, distribution, None, for_py_version) + if version_spec: + specifier: Final[Specifier] = Specifier(version_spec) + if specifier.operator not in {"==", "<"}: raise ValueError(version_spec) - version = Wheel.as_version_tuple(version_spec[from_pos:]) - start = next((at for at, w in enumerate(wheels) if op(w.version_tuple, version)), len(wheels)) + return next((wheel for wheel in wheels if specifier.contains(wheel.version)), None) + return next(iter(wheels), None) - return None if start == end else wheels[start] - -def pip_wheel_env_run(search_dirs, app_data, env): +def pip_wheel_env_run(search_dirs: list[Path], app_data: AppData, env: dict[str, str]) -> dict[str, str]: env = env.copy() env.update({"PIP_USE_WHEEL": "1", "PIP_USER": "0", "PIP_NO_INPUT": "1", "PYTHONIOENCODING": "utf-8"}) wheel = get_wheel( @@ -127,8 +217,23 @@ def pip_wheel_env_run(search_dirs, app_data, env): return env +def _check_distribution(distribution: str) -> None: + if not _DISTRIBUTION_RE.fullmatch(distribution): + msg = f"refusing to download wheel for suspicious distribution name: {distribution!r}" + raise ValueError(msg) + + +def _check_version_spec(version_spec: str | None) -> None: + if not version_spec: + return + if not _VERSION_SPEC_RE.fullmatch(version_spec): + msg = f"refusing to download wheel with suspicious version spec: {version_spec!r}" + raise ValueError(msg) + + __all__ = [ "download_wheel", + "find_compatible_in_house", "get_wheel", "pip_wheel_env_run", ] diff --git a/src/virtualenv/seed/wheels/bundle.py b/src/virtualenv/seed/wheels/bundle.py index 523e45ca2..948f7d018 100644 --- a/src/virtualenv/seed/wheels/bundle.py +++ b/src/virtualenv/seed/wheels/bundle.py @@ -1,12 +1,27 @@ from __future__ import annotations +from typing import TYPE_CHECKING + from virtualenv.seed.wheels.embed import get_embed_wheel from .periodic_update import periodic_update from .util import Version, Wheel, discover_wheels +if TYPE_CHECKING: + from pathlib import Path + + from virtualenv.app_data.base import AppData + -def from_bundle(distribution, version, for_py_version, search_dirs, app_data, do_periodic_update, env): # noqa: PLR0913 +def from_bundle( # ruff:ignore[too-many-arguments] + distribution: str, + version: str | None, + for_py_version: str, + search_dirs: list[Path], + app_data: AppData, + do_periodic_update: bool, + env: dict[str, str], +) -> Wheel | None: """Load the bundled wheel to a cache directory.""" of_version = Version.of_version(version) wheel = load_embed_wheel(app_data, distribution, for_py_version, of_version) @@ -19,24 +34,24 @@ def from_bundle(distribution, version, for_py_version, search_dirs, app_data, do # 3. acquire from extra search dir found_wheel = from_dir(distribution, of_version, for_py_version, search_dirs) - if found_wheel is not None and (wheel is None or found_wheel.version_tuple > wheel.version_tuple): + if found_wheel is not None and (wheel is None or found_wheel.parsed_version > wheel.parsed_version): wheel = found_wheel return wheel -def load_embed_wheel(app_data, distribution, for_py_version, version): +def load_embed_wheel(app_data: AppData, distribution: str, for_py_version: str, version: str | None) -> Wheel | None: wheel = get_embed_wheel(distribution, for_py_version) if wheel is not None: version_match = version == wheel.version if version is None or version_match: - with app_data.ensure_extracted(wheel.path, lambda: app_data.house) as wheel_path: + with app_data.ensure_extracted(wheel.path, lambda: app_data.house) as wheel_path: # ty: ignore[invalid-argument-type] wheel = Wheel(wheel_path) else: # if version does not match ignore wheel = None return wheel -def from_dir(distribution, version, for_py_version, directories): +def from_dir(distribution: str, version: str | None, for_py_version: str, directories: list[Path]) -> Wheel | None: """Load a compatible wheel from a given folder.""" for folder in directories: for wheel in discover_wheels(folder, distribution, version, for_py_version): diff --git a/src/virtualenv/seed/wheels/embed/__init__.py b/src/virtualenv/seed/wheels/embed/__init__.py index ef6f78e86..1d8be7311 100644 --- a/src/virtualenv/seed/wheels/embed/__init__.py +++ b/src/virtualenv/seed/wheels/embed/__init__.py @@ -1,60 +1,135 @@ from __future__ import annotations +import hashlib +import zipfile from pathlib import Path +from virtualenv.info import IS_ZIPAPP, ROOT from virtualenv.seed.wheels.util import Wheel BUNDLE_FOLDER = Path(__file__).absolute().parent BUNDLE_SUPPORT = { - "3.8": { - "pip": "pip-25.0.1-py3-none-any.whl", - "setuptools": "setuptools-75.3.2-py3-none-any.whl", - "wheel": "wheel-0.45.1-py3-none-any.whl", - }, "3.9": { - "pip": "pip-25.3-py3-none-any.whl", - "setuptools": "setuptools-80.9.0-py3-none-any.whl", + "pip": "pip-26.0.1-py3-none-any.whl", + "setuptools": "setuptools-82.0.1-py3-none-any.whl", }, "3.10": { - "pip": "pip-25.3-py3-none-any.whl", - "setuptools": "setuptools-80.9.0-py3-none-any.whl", + "pip": "pip-26.2.1-py3-none-any.whl", + "setuptools": "setuptools-84.0.0-py3-none-any.whl", }, "3.11": { - "pip": "pip-25.3-py3-none-any.whl", - "setuptools": "setuptools-80.9.0-py3-none-any.whl", + "pip": "pip-26.2.1-py3-none-any.whl", + "setuptools": "setuptools-84.0.0-py3-none-any.whl", }, "3.12": { - "pip": "pip-25.3-py3-none-any.whl", - "setuptools": "setuptools-80.9.0-py3-none-any.whl", + "pip": "pip-26.2.1-py3-none-any.whl", + "setuptools": "setuptools-84.0.0-py3-none-any.whl", }, "3.13": { - "pip": "pip-25.3-py3-none-any.whl", - "setuptools": "setuptools-80.9.0-py3-none-any.whl", + "pip": "pip-26.2.1-py3-none-any.whl", + "setuptools": "setuptools-84.0.0-py3-none-any.whl", }, "3.14": { - "pip": "pip-25.3-py3-none-any.whl", - "setuptools": "setuptools-80.9.0-py3-none-any.whl", + "pip": "pip-26.2.1-py3-none-any.whl", + "setuptools": "setuptools-84.0.0-py3-none-any.whl", }, "3.15": { - "pip": "pip-25.3-py3-none-any.whl", - "setuptools": "setuptools-80.9.0-py3-none-any.whl", + "pip": "pip-26.2.1-py3-none-any.whl", + "setuptools": "setuptools-84.0.0-py3-none-any.whl", + }, + "3.16": { + "pip": "pip-26.2.1-py3-none-any.whl", + "setuptools": "setuptools-84.0.0-py3-none-any.whl", }, } -MAX = "3.8" +MAX = next(reversed(BUNDLE_SUPPORT)) +MIN = next(iter(BUNDLE_SUPPORT)) + + +def _release_tuple(version: str) -> tuple[int, ...]: + return tuple(int(part) for part in version.split(".")) + + +# oldest target Python version virtualenv still bundles seed wheels for; anything below this has no embedded pip +OLDEST_SUPPORTED = _release_tuple(MIN) + +# SHA-256 of every bundled wheel. Verified on load so a corrupted or tampered wheel on disk fails loud instead of +# being handed to pip. Generated together with ``BUNDLE_SUPPORT`` by ``tasks/upgrade_wheels.py``. +BUNDLE_SHA256 = { + "pip-26.0.1-py3-none-any.whl": "bdb1b08f4274833d62c1aa29e20907365a2ceb950410df15fc9521bad440122b", + "pip-26.2.1-py3-none-any.whl": "71138adf1f4ca900cdb7d289c21b7494329f2332b6d85f0e1c42108c0384ed3e", + "setuptools-82.0.1-py3-none-any.whl": "a59e362652f08dcd477c78bb6e7bd9d80a7995bc73ce773050228a348ce2e5bb", + "setuptools-84.0.0-py3-none-any.whl": "51a52592b3b99e102b609654876bd65f19f999935166d1352678931132b0c670", +} + +_VERIFIED_WHEELS: set[str] = set() -def get_embed_wheel(distribution, for_py_version): - mapping = BUNDLE_SUPPORT.get(for_py_version, {}) or BUNDLE_SUPPORT[MAX] +def get_embed_wheel(distribution: str, for_py_version: str | None) -> Wheel | None: + """Return the bundled wheel that ships with virtualenv for a given distribution and Python version. + + :param distribution: project name of the seed package, for example ``pip`` or ``setuptools``. + :param for_py_version: major.minor Python version string the environment will be created for, or ``None`` to use the + newest bundle. + + :returns: a :class:`Wheel` pointing at the verified bundled file, or ``None`` when no wheel is bundled for the + requested combination, including target versions below the oldest bundled one. + + :raises RuntimeError: if the bundled wheel on disk fails SHA-256 verification. + + """ + if for_py_version is None or _release_tuple(for_py_version) > _release_tuple(MAX): + # no specific target, or a Python newer than anything bundled: reuse the newest bundle + mapping = BUNDLE_SUPPORT[MAX] + else: # versions below the oldest bundled one fall through to None instead of an incompatible newer wheel + mapping = BUNDLE_SUPPORT.get(for_py_version) + if not mapping: + return None wheel_file = mapping.get(distribution) if wheel_file is None: return None path = BUNDLE_FOLDER / wheel_file + _verify_bundled_wheel(path) return Wheel.from_path(path) +def _verify_bundled_wheel(path: Path) -> None: + name = path.name + if name in _VERIFIED_WHEELS: + return + expected = BUNDLE_SHA256.get(name) + if expected is None: + msg = f"bundled wheel {name} has no recorded sha256 in BUNDLE_SHA256" + raise RuntimeError(msg) + actual = _hash_bundled_wheel(path) + if actual != expected: + msg = f"bundled wheel {name} sha256 mismatch: expected {expected}, got {actual}" + raise RuntimeError(msg) + _VERIFIED_WHEELS.add(name) + + +def _hash_bundled_wheel(path: Path) -> str: + # ``path`` is under the package directory; when virtualenv runs from a zipapp the wheel lives inside the + # archive and cannot be opened as a regular file, so read the bytes straight from the zipapp entry. + digest = hashlib.sha256() + if IS_ZIPAPP: + entry = path.resolve().relative_to(Path(ROOT).resolve()).as_posix() + with zipfile.ZipFile(ROOT, "r") as archive, archive.open(entry) as stream: + for chunk in iter(lambda: stream.read(1 << 20), b""): + digest.update(chunk) + else: + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + __all__ = [ "BUNDLE_FOLDER", + "BUNDLE_SHA256", "BUNDLE_SUPPORT", "MAX", + "MIN", + "OLDEST_SUPPORTED", "get_embed_wheel", ] diff --git a/src/virtualenv/seed/wheels/embed/pip-25.0.1-py3-none-any.whl b/src/virtualenv/seed/wheels/embed/pip-25.0.1-py3-none-any.whl deleted file mode 100644 index 8d3b0043e..000000000 Binary files a/src/virtualenv/seed/wheels/embed/pip-25.0.1-py3-none-any.whl and /dev/null differ diff --git a/src/virtualenv/seed/wheels/embed/pip-25.3-py3-none-any.whl b/src/virtualenv/seed/wheels/embed/pip-26.0.1-py3-none-any.whl similarity index 70% rename from src/virtualenv/seed/wheels/embed/pip-25.3-py3-none-any.whl rename to src/virtualenv/seed/wheels/embed/pip-26.0.1-py3-none-any.whl index 755e1aa0c..580d09a92 100644 Binary files a/src/virtualenv/seed/wheels/embed/pip-25.3-py3-none-any.whl and b/src/virtualenv/seed/wheels/embed/pip-26.0.1-py3-none-any.whl differ diff --git a/src/virtualenv/seed/wheels/embed/pip-26.2.1-py3-none-any.whl b/src/virtualenv/seed/wheels/embed/pip-26.2.1-py3-none-any.whl new file mode 100644 index 000000000..bc442f6b9 Binary files /dev/null and b/src/virtualenv/seed/wheels/embed/pip-26.2.1-py3-none-any.whl differ diff --git a/src/virtualenv/seed/wheels/embed/setuptools-75.3.2-py3-none-any.whl b/src/virtualenv/seed/wheels/embed/setuptools-75.3.2-py3-none-any.whl deleted file mode 100644 index 1b66a67ff..000000000 Binary files a/src/virtualenv/seed/wheels/embed/setuptools-75.3.2-py3-none-any.whl and /dev/null differ diff --git a/src/virtualenv/seed/wheels/embed/setuptools-80.9.0-py3-none-any.whl b/src/virtualenv/seed/wheels/embed/setuptools-80.9.0-py3-none-any.whl deleted file mode 100644 index 2412ad4a3..000000000 Binary files a/src/virtualenv/seed/wheels/embed/setuptools-80.9.0-py3-none-any.whl and /dev/null differ diff --git a/src/virtualenv/seed/wheels/embed/setuptools-82.0.1-py3-none-any.whl b/src/virtualenv/seed/wheels/embed/setuptools-82.0.1-py3-none-any.whl new file mode 100644 index 000000000..2db749a77 Binary files /dev/null and b/src/virtualenv/seed/wheels/embed/setuptools-82.0.1-py3-none-any.whl differ diff --git a/src/virtualenv/seed/wheels/embed/setuptools-84.0.0-py3-none-any.whl b/src/virtualenv/seed/wheels/embed/setuptools-84.0.0-py3-none-any.whl new file mode 100644 index 000000000..f38c87a39 Binary files /dev/null and b/src/virtualenv/seed/wheels/embed/setuptools-84.0.0-py3-none-any.whl differ diff --git a/src/virtualenv/seed/wheels/embed/wheel-0.45.1-py3-none-any.whl b/src/virtualenv/seed/wheels/embed/wheel-0.45.1-py3-none-any.whl deleted file mode 100644 index 589308a21..000000000 Binary files a/src/virtualenv/seed/wheels/embed/wheel-0.45.1-py3-none-any.whl and /dev/null differ diff --git a/src/virtualenv/seed/wheels/periodic_update.py b/src/virtualenv/seed/wheels/periodic_update.py index ac627b3bd..8481f3ea3 100644 --- a/src/virtualenv/seed/wheels/periodic_update.py +++ b/src/virtualenv/seed/wheels/periodic_update.py @@ -2,11 +2,13 @@ from __future__ import annotations +import hashlib import json import logging import os import ssl import sys +from contextlib import suppress from datetime import datetime, timedelta, timezone from itertools import groupby from pathlib import Path @@ -14,6 +16,8 @@ from subprocess import DEVNULL, Popen from textwrap import dedent from threading import Thread +from time import sleep +from typing import TYPE_CHECKING, Final from urllib.error import URLError from urllib.request import urlopen @@ -22,6 +26,11 @@ from virtualenv.seed.wheels.util import Wheel from virtualenv.util.subprocess import CREATE_NO_WINDOW +if TYPE_CHECKING: + from collections.abc import Generator + + from virtualenv.app_data.base import AppData, ContentStore + LOGGER = logging.getLogger(__name__) GRACE_PERIOD_CI = timedelta(hours=1) # prevent version switch in the middle of a CI run GRACE_PERIOD_MINOR = timedelta(days=28) @@ -29,22 +38,22 @@ UPDATE_ABORTED_DELAY = timedelta(hours=1) -def periodic_update( # noqa: PLR0913 - distribution, - of_version, - for_py_version, - wheel, - search_dirs, - app_data, - do_periodic_update, - env, -): +def periodic_update( # ruff:ignore[too-many-arguments] + distribution: str, + of_version: str | None, + for_py_version: str, + wheel: Wheel | None, + search_dirs: list[Path], + app_data: AppData, + do_periodic_update: bool, + env: dict[str, str], +) -> Wheel | None: if do_periodic_update: handle_auto_update(distribution, for_py_version, wheel, search_dirs, app_data, env) now = datetime.now(tz=timezone.utc) - def _update_wheel(ver): + def _update_wheel(ver: NewVersion) -> Wheel: updated_wheel = Wheel(app_data.house / ver.filename) LOGGER.debug("using %supdated wheel %s", "periodically " if updated_wheel else "", updated_wheel) return updated_wheel @@ -68,22 +77,28 @@ def _update_wheel(ver): return wheel -def handle_auto_update(distribution, for_py_version, wheel, search_dirs, app_data, env): # noqa: PLR0913 - embed_update_log = app_data.embed_update_log(distribution, for_py_version) - u_log = UpdateLog.from_dict(embed_update_log.read()) +def handle_auto_update( # ruff:ignore[too-many-arguments] + distribution: str, + for_py_version: str, + wheel: Wheel | None, + search_dirs: list[Path], + app_data: AppData, + env: dict[str, str], +) -> None: + u_log = UpdateLog.from_app_data(app_data, distribution, for_py_version) if u_log.needs_update: u_log.periodic = True u_log.started = datetime.now(tz=timezone.utc) - embed_update_log.write(u_log.to_dict()) + app_data.embed_update_log(distribution, for_py_version).write(u_log.to_dict()) trigger_update(distribution, for_py_version, wheel, search_dirs, app_data, periodic=True, env=env) -def add_wheel_to_update_log(wheel, for_py_version, app_data): +def add_wheel_to_update_log(wheel: Wheel, for_py_version: str, app_data: AppData) -> None: embed_update_log = app_data.embed_update_log(wheel.distribution, for_py_version) - LOGGER.debug("adding %s information to %s", wheel.name, embed_update_log.file) - u_log = UpdateLog.from_dict(embed_update_log.read()) + LOGGER.debug("adding %s information to %s", wheel.name, embed_update_log.file) # ty: ignore[unresolved-attribute] + u_log = UpdateLog.from_app_data(app_data, wheel.distribution, for_py_version) if any(version.filename == wheel.name for version in u_log.versions): - LOGGER.warning("%s already present in %s", wheel.name, embed_update_log.file) + LOGGER.warning("%s already present in %s", wheel.name, embed_update_log.file) # ty: ignore[unresolved-attribute] return # we don't need a release date for sources other than "periodic" version = NewVersion(wheel.name, datetime.now(tz=timezone.utc), None, "download") @@ -94,31 +109,31 @@ def add_wheel_to_update_log(wheel, for_py_version, app_data): DATETIME_FMT = "%Y-%m-%dT%H:%M:%S.%fZ" -def dump_datetime(value): +def dump_datetime(value: datetime | None) -> str | None: return None if value is None else value.strftime(DATETIME_FMT) -def load_datetime(value): +def load_datetime(value: str | None) -> datetime | None: return None if value is None else datetime.strptime(value, DATETIME_FMT).replace(tzinfo=timezone.utc) -class NewVersion: # noqa: PLW1641 - def __init__(self, filename, found_date, release_date, source) -> None: +class NewVersion: # ruff:ignore[eq-without-hash] + def __init__(self, filename: str, found_date: datetime, release_date: datetime | None, source: str) -> None: self.filename = filename self.found_date = found_date self.release_date = release_date self.source = source @classmethod - def from_dict(cls, dictionary): + def from_dict(cls, dictionary: dict[str, str | None]) -> NewVersion: return cls( - filename=dictionary["filename"], - found_date=load_datetime(dictionary["found_date"]), + filename=dictionary["filename"], # ty: ignore[invalid-argument-type] + found_date=load_datetime(dictionary["found_date"]), # ty: ignore[invalid-argument-type] release_date=load_datetime(dictionary["release_date"]), - source=dictionary["source"], + source=dictionary["source"], # ty: ignore[invalid-argument-type] ) - def to_dict(self): + def to_dict(self) -> dict[str, str | None]: return { "filename": self.filename, "release_date": dump_datetime(self.release_date), @@ -126,7 +141,7 @@ def to_dict(self): "source": self.source, } - def use(self, now, ignore_grace_period_minor=False, ignore_grace_period_ci=False): # noqa: FBT002 + def use(self, now: datetime, ignore_grace_period_minor: bool = False, ignore_grace_period_ci: bool = False) -> bool: # ruff:ignore[boolean-default-value-positional-argument] if self.source == "manual": return True if self.source == "periodic" and (self.found_date < now - GRACE_PERIOD_CI or ignore_grace_period_ci): @@ -142,43 +157,53 @@ def __repr__(self) -> str: f"release_date={self.release_date}, source={self.source})" ) - def __eq__(self, other): - return type(self) == type(other) and all( # noqa: E721 + def __eq__(self, other: object) -> bool: + return type(self) == type(other) and all( # ruff:ignore[type-comparison] getattr(self, k) == getattr(other, k) for k in ["filename", "release_date", "found_date", "source"] ) - def __ne__(self, other): + def __ne__(self, other: object) -> bool: return not (self == other) @property - def wheel(self): + def wheel(self) -> Wheel: return Wheel(Path(self.filename)) class UpdateLog: - def __init__(self, started, completed, versions, periodic) -> None: + def __init__( + self, started: datetime | None, completed: datetime | None, versions: list[NewVersion], periodic: bool | None + ) -> None: self.started = started self.completed = completed self.versions = versions self.periodic = periodic @classmethod - def from_dict(cls, dictionary): + def from_dict(cls, dictionary: dict[str, object] | None) -> UpdateLog: if dictionary is None: dictionary = {} return cls( - load_datetime(dictionary.get("started")), - load_datetime(dictionary.get("completed")), - [NewVersion.from_dict(v) for v in dictionary.get("versions", [])], - dictionary.get("periodic"), + load_datetime(dictionary.get("started")), # ty: ignore[invalid-argument-type] + load_datetime(dictionary.get("completed")), # ty: ignore[invalid-argument-type] + [NewVersion.from_dict(v) for v in dictionary.get("versions", [])], # ty: ignore[not-iterable] + dictionary.get("periodic"), # ty: ignore[invalid-argument-type] ) @classmethod - def from_app_data(cls, app_data, distribution, for_py_version): - raw_json = app_data.embed_update_log(distribution, for_py_version).read() - return cls.from_dict(raw_json) - - def to_dict(self): + def from_app_data(cls, app_data: AppData, distribution: str, for_py_version: str) -> UpdateLog: + store: Final[ContentStore] = app_data.embed_update_log(distribution, for_py_version) + try: + return cls.from_dict(store.read()) + except (AttributeError, KeyError, TypeError, ValueError): + # another virtualenv version or an interrupted run can leave valid JSON of the wrong shape in the shared + # app data; drop it as read() drops invalid JSON, since an empty log falls back to the bundled wheel + LOGGER.warning("removing malformed embed update log of %s", distribution) + with suppress(OSError): # another process may remove or rewrite the file at the same time + store.remove() + return cls(None, None, [], None) + + def to_dict(self) -> dict[str, object]: return { "started": dump_datetime(self.started), "completed": dump_datetime(self.completed), @@ -187,7 +212,7 @@ def to_dict(self): } @property - def needs_update(self): + def needs_update(self) -> bool: now = datetime.now(tz=timezone.utc) if self.completed is None: # never completed return self._check_start(now) @@ -195,11 +220,19 @@ def needs_update(self): return False return self._check_start(now) - def _check_start(self, now): + def _check_start(self, now: datetime) -> bool: return self.started is None or now - self.started > UPDATE_ABORTED_DELAY -def trigger_update(distribution, for_py_version, wheel, search_dirs, app_data, env, periodic): # noqa: PLR0913 +def trigger_update( # ruff:ignore[too-many-arguments] + distribution: str, + for_py_version: str, + wheel: Wheel | None, + search_dirs: list[Path], + app_data: AppData, + env: dict[str, str], + periodic: bool, +) -> None: wheel_path = None if wheel is None else str(wheel.path) cmd = [ sys.executable, @@ -220,7 +253,7 @@ def trigger_update(distribution, for_py_version, wheel, search_dirs, app_data, e kwargs = {"stdout": pipe, "stderr": pipe} if not debug and sys.platform == "win32": kwargs["creationflags"] = CREATE_NO_WINDOW - process = Popen(cmd, **kwargs) + process = Popen(cmd, **kwargs) # ty: ignore[no-matching-overload] LOGGER.info( "triggered periodic upgrade of %s%s (for python %s) via background process having PID %d", distribution, @@ -235,7 +268,14 @@ def trigger_update(distribution, for_py_version, wheel, search_dirs, app_data, e process.returncode = 0 -def do_update(distribution, for_py_version, embed_filename, app_data, search_dirs, periodic): # noqa: PLR0913 +def do_update( # ruff:ignore[too-many-arguments] + distribution: str, + for_py_version: str, + embed_filename: str | None, + app_data: str | AppData, + search_dirs: list[str] | list[Path], + periodic: bool, +) -> list[NewVersion] | None: versions = None try: versions = _run_do_update(app_data, distribution, embed_filename, for_py_version, periodic, search_dirs) @@ -244,23 +284,22 @@ def do_update(distribution, for_py_version, embed_filename, app_data, search_dir return versions -def _run_do_update( # noqa: C901, PLR0913 - app_data, - distribution, - embed_filename, - for_py_version, - periodic, - search_dirs, -): - from virtualenv.seed.wheels import acquire # noqa: PLC0415 +def _run_do_update( # ruff:ignore[complex-structure, too-many-arguments] + app_data: str | AppData, + distribution: str, + embed_filename: str | None, + for_py_version: str, + periodic: bool, + search_dirs: list[str] | list[Path], +) -> list[NewVersion]: + from virtualenv.seed.wheels import acquire # ruff:ignore[import-outside-top-level] wheel_filename = None if embed_filename is None else Path(embed_filename) - embed_version = None if wheel_filename is None else Wheel(wheel_filename).version_tuple + embed_version = None if wheel_filename is None else Wheel(wheel_filename).parsed_version app_data = AppDataDiskFolder(app_data) if isinstance(app_data, str) else app_data search_dirs = [Path(p) if isinstance(p, str) else p for p in search_dirs] wheelhouse = app_data.house - embed_update_log = app_data.embed_update_log(distribution, for_py_version) - u_log = UpdateLog.from_dict(embed_update_log.read()) + u_log = UpdateLog.from_app_data(app_data, distribution, for_py_version) now = datetime.now(tz=timezone.utc) update_versions, other_versions = [], [] @@ -292,7 +331,7 @@ def _run_do_update( # noqa: C901, PLR0913 search_dirs=search_dirs, app_data=app_data, to_folder=wheelhouse, - env=os.environ, + env=os.environ, # ty: ignore[invalid-argument-type] ) if dest is None or (update_versions and update_versions[0].filename == dest.name): break @@ -303,7 +342,7 @@ def _run_do_update( # noqa: C901, PLR0913 filenames.add(last.filename) last_wheel = last.wheel last_version = last_wheel.version - if embed_version is not None and embed_version >= last_wheel.version_tuple: + if embed_version is not None and embed_version >= last_wheel.parsed_version: break # stop download if we reach the embed version u_log.periodic = periodic if not u_log.periodic: @@ -312,55 +351,119 @@ def _run_do_update( # noqa: C901, PLR0913 other_versions = [version for version in other_versions if version.filename not in filenames] u_log.versions = versions + update_versions + other_versions u_log.completed = datetime.now(tz=timezone.utc) - embed_update_log.write(u_log.to_dict()) + app_data.embed_update_log(distribution, for_py_version).write(u_log.to_dict()) return versions -def release_date_for_wheel_path(dest): +def release_date_for_wheel_path(dest: Path) -> datetime | None: wheel = Wheel(dest) # the most accurate is to ask PyPi - e.g. https://pypi.org/pypi/pip/json, # see https://warehouse.pypa.io/api-reference/json/ for more details content = _pypi_get_distribution_info_cached(wheel.distribution) if content is not None: try: - upload_time = content["releases"][wheel.version][0]["upload_time"] + upload_time = content["releases"][wheel.version][0]["upload_time"] # ty: ignore[not-subscriptable] return datetime.strptime(upload_time, "%Y-%m-%dT%H:%M:%S").replace(tzinfo=timezone.utc) - except Exception as exception: # noqa: BLE001 - LOGGER.error("could not load release date %s because %r", content, exception) # noqa: TRY400 + except Exception as exception: # ruff:ignore[blind-except] + LOGGER.error("could not load release date %s because %r", content, exception) # ruff:ignore[error-instead-of-exception] return None -def _request_context(): +#: Pauses between PyPI metadata lookups before refusing a wheel we could not verify, to ride out a brief outage or a JSON +#: API that has not caught up with a fresh upload. +_DIGEST_RETRY_DELAYS: Final[tuple[float, ...]] = (1.0, 2.0, 4.0) + + +def verify_wheel_digest(wheel: Wheel) -> None: + """Verify a downloaded wheel's sha256 against the digest PyPI's JSON API reports for that release. + + ``pip download`` trusts whatever index it is configured to use, and nothing else in this module checks the bytes it + hands back. Callers run this only when pip used the default index, so PyPI served the file and must list its digest. + A failed metadata request, an unknown filename or a missing sha256 therefore fails like a mismatch: accepting the + wheel in those cases would let anyone who can block ``pypi.org/pypi`` turn the check off. + + :raises UnverifiedWheelError: if PyPI's sha256 for this exact filename cannot be obtained after retrying. + :raises RuntimeError: if the sha256 does not match. + + """ + expected = _pypi_sha256_for_wheel(wheel) + for delay in _DIGEST_RETRY_DELAYS: + if expected is not None: + break + LOGGER.warning("could not obtain the sha256 of %s from PyPI, retrying in %s seconds", wheel.name, delay) + _PYPI_CACHE.pop(wheel.distribution, None) + sleep(delay) + expected = _pypi_sha256_for_wheel(wheel) + if expected is None: + msg = f"refusing downloaded wheel {wheel.name}: could not obtain its sha256 from PyPI to verify it" + raise UnverifiedWheelError(msg) + if (actual := hashlib.sha256(wheel.path.read_bytes()).hexdigest()) != expected: + msg = f"downloaded wheel {wheel.name} has sha256 {actual}, but PyPI reports {expected} for this release" + raise RuntimeError(msg) + + +class UnverifiedWheelError(RuntimeError): + """PyPI's digest for a downloaded wheel was unavailable, so a caller may fall back to the bundled wheel.""" + + +def _pypi_sha256_for_wheel(wheel: Wheel) -> str | None: + content = _pypi_get_distribution_info_cached(wheel.distribution) + releases = content.get("releases") if isinstance(content, dict) else None + entries = releases.get(wheel.version) if isinstance(releases, dict) else None + if not isinstance(entries, list): + return None + entry = next((entry for entry in entries if isinstance(entry, dict) and entry.get("filename") == wheel.name), {}) + digests = entry.get("digests") + sha256 = digests.get("sha256") if isinstance(digests, dict) else None + return sha256 if isinstance(sha256, str) else None + + +#: Opt-in escape hatch to restore the pre-2026 behavior of falling back to an unverified HTTPS context when the +#: verified request fails. Off by default: a failed TLS handshake on the PyPI metadata lookup now aborts the update +#: instead of silently downgrading, because the response drives which wheel version virtualenv thinks is up to date. +_INSECURE_FALLBACK_ENV = "VIRTUALENV_PERIODIC_UPDATE_INSECURE" + + +def _request_context() -> Generator[ssl.SSLContext | None, None, None]: yield None - # fallback to non verified HTTPS (the information we request is not sensitive, so fallback) - yield ssl._create_unverified_context() # noqa: S323, SLF001 + if os.environ.get(_INSECURE_FALLBACK_ENV): + LOGGER.warning( + "falling back to unverified HTTPS for PyPI metadata because %s is set", + _INSECURE_FALLBACK_ENV, + ) + yield ssl._create_unverified_context() # ruff:ignore[suspicious-unverified-context-usage, private-member-access] _PYPI_CACHE = {} -def _pypi_get_distribution_info_cached(distribution): +def _pypi_get_distribution_info_cached(distribution: str) -> dict[str, object] | None: if distribution not in _PYPI_CACHE: _PYPI_CACHE[distribution] = _pypi_get_distribution_info(distribution) return _PYPI_CACHE[distribution] -def _pypi_get_distribution_info(distribution): +def _pypi_get_distribution_info(distribution: str) -> dict[str, object] | None: content, url = None, f"https://pypi.org/pypi/{distribution}/json" try: for context in _request_context(): - try: - with urlopen(url, context=context) as file_handler: # noqa: S310 - content = json.load(file_handler) + if (content := _fetch_pypi_json(url, context)) is not None: break - except URLError as exception: - LOGGER.error("failed to access %s because %r", url, exception) # noqa: TRY400 - except Exception as exception: # noqa: BLE001 - LOGGER.error("failed to access %s because %r", url, exception) # noqa: TRY400 + except Exception as exception: # ruff:ignore[blind-except] + LOGGER.error("failed to access %s because %r", url, exception) # ruff:ignore[error-instead-of-exception] return content -def manual_upgrade(app_data, env): +def _fetch_pypi_json(url: str, context: ssl.SSLContext | None) -> dict[str, object] | None: + try: + with urlopen(url, context=context) as file_handler: # ruff:ignore[suspicious-url-open-usage] + return json.load(file_handler) + except URLError as exception: + LOGGER.error("failed to access %s because %r", url, exception) # ruff:ignore[error-instead-of-exception] + return None + + +def manual_upgrade(app_data: AppData, env: dict[str, str]) -> None: threads = [] for for_py_version, distribution_to_package in BUNDLE_SUPPORT.items(): @@ -374,9 +477,9 @@ def manual_upgrade(app_data, env): thread.join() -def _run_manual_upgrade(app_data, distribution, for_py_version, env): +def _run_manual_upgrade(app_data: AppData, distribution: str, for_py_version: str, env: dict[str, str]) -> None: start = datetime.now(tz=timezone.utc) - from .bundle import from_bundle # noqa: PLC0415 + from .bundle import from_bundle # ruff:ignore[import-outside-top-level] current = from_bundle( distribution=distribution, @@ -396,7 +499,7 @@ def _run_manual_upgrade(app_data, distribution, for_py_version, env): versions = do_update( distribution=distribution, for_py_version=for_py_version, - embed_filename=current.path, + embed_filename=current.path, # ty: ignore[invalid-argument-type, unresolved-attribute] app_data=app_data, search_dirs=[], periodic=False, @@ -416,6 +519,7 @@ def _run_manual_upgrade(app_data, distribution, for_py_version, env): __all__ = [ "NewVersion", + "UnverifiedWheelError", "UpdateLog", "add_wheel_to_update_log", "do_update", @@ -425,4 +529,5 @@ def _run_manual_upgrade(app_data, distribution, for_py_version, env): "periodic_update", "release_date_for_wheel_path", "trigger_update", + "verify_wheel_digest", ] diff --git a/src/virtualenv/seed/wheels/util.py b/src/virtualenv/seed/wheels/util.py index 2bc01ae27..d693cca69 100644 --- a/src/virtualenv/seed/wheels/util.py +++ b/src/virtualenv/seed/wheels/util.py @@ -1,76 +1,86 @@ from __future__ import annotations +import logging +from email.parser import HeaderParser from operator import attrgetter +from typing import TYPE_CHECKING, Final from zipfile import ZipFile +from packaging.specifiers import InvalidSpecifier, SpecifierSet +from packaging.version import Version as PackagingVersion + +if TYPE_CHECKING: + from pathlib import Path + +_LOGGER: Final[logging.Logger] = logging.getLogger(__name__) +# https://packaging.python.org/en/latest/specifications/binary-distribution-format/#file-name-convention +# {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}-{platform tag}.whl +_MIN_WHEEL_NAME_PARTS: Final[int] = 5 + + +def discover_wheels(from_folder: Path, distribution: str, version: str | None, for_py_version: str) -> list[Wheel]: + return sorted( + ( + wheel + for filename in from_folder.iterdir() + if (wheel := Wheel.from_path(filename)) + and wheel.distribution == distribution + and (version is None or wheel.version == version) + and wheel.support_py(for_py_version) + ), + key=attrgetter("parsed_version", "distribution"), + reverse=True, + ) + class Wheel: - def __init__(self, path) -> None: - # https://www.python.org/dev/peps/pep-0427/#file-name-convention - # The wheel filename is {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}-{platform tag}.whl + def __init__(self, path: Path) -> None: self.path = path self._parts = path.stem.split("-") @classmethod - def from_path(cls, path): - if path is not None and path.suffix == ".whl" and len(path.stem.split("-")) >= 5: # noqa: PLR2004 + def from_path(cls, path: Path) -> Wheel | None: + if path.suffix == ".whl" and len(path.stem.split("-")) >= _MIN_WHEEL_NAME_PARTS: return cls(path) return None + def support_py(self, py_version: str) -> bool: + """Check whether ``Requires-Python`` admits ``py_version``, a ``major.minor`` string.""" + name = f"{self.distribution}-{self.version}.dist-info/METADATA" + with ZipFile(str(self.path), "r") as zip_file: + requires = HeaderParser().parsestr(zip_file.read(name).decode("utf-8")).get("Requires-Python") + if requires is None: + return True + try: + return SpecifierSet(requires).contains(py_version) + except InvalidSpecifier: + # skip the wheel as pip does, rather than abort seeding or override the verified embedded wheel + _LOGGER.warning("skip %s, its Requires-Python %r is not a valid specifier", self.path, requires) + return False + @property - def distribution(self): + def distribution(self) -> str: return self._parts[0] @property - def version(self): + def version(self) -> str: return self._parts[1] @property - def version_tuple(self): + def version_tuple(self) -> tuple[int, ...]: return self.as_version_tuple(self.version) @staticmethod - def as_version_tuple(version): - result = [] - for part in version.split(".")[0:3]: - try: - result.append(int(part)) - except ValueError: # noqa: PERF203 - break - if not result: - raise ValueError(version) - return tuple(result) + def as_version_tuple(version: str) -> tuple[int, ...]: + return PackagingVersion(version).release @property - def name(self): - return self.path.name + def parsed_version(self) -> PackagingVersion: + return PackagingVersion(self.version) - def support_py(self, py_version): - name = f"{'-'.join(self.path.stem.split('-')[0:2])}.dist-info/METADATA" - with ZipFile(str(self.path), "r") as zip_file: - metadata = zip_file.read(name).decode("utf-8") - marker = "Requires-Python:" - requires = next((i[len(marker) :] for i in metadata.splitlines() if i.startswith(marker)), None) - if requires is None: # if it does not specify a python requires the assumption is compatible - return True - py_version_int = tuple(int(i) for i in py_version.split(".")) - for require in (i.strip() for i in requires.split(",")): - # https://www.python.org/dev/peps/pep-0345/#version-specifiers - for operator, check in [ - ("!=", lambda v: py_version_int != v), - ("==", lambda v: py_version_int == v), - ("<=", lambda v: py_version_int <= v), - (">=", lambda v: py_version_int >= v), - ("<", lambda v: py_version_int < v), - (">", lambda v: py_version_int > v), - ]: - if require.startswith(operator): - ver_str = require[len(operator) :].strip() - version = tuple((int(i) if i != "*" else None) for i in ver_str.split("."))[0:2] - if not check(version): - return False - break - return True + @property + def name(self) -> str: + return self.path.name def __repr__(self) -> str: return f"{self.__class__.__name__}({self.path})" @@ -79,39 +89,24 @@ def __str__(self) -> str: return str(self.path) -def discover_wheels(from_folder, distribution, version, for_py_version): - wheels = [] - for filename in from_folder.iterdir(): - wheel = Wheel.from_path(filename) - if ( - wheel - and wheel.distribution == distribution - and (version is None or wheel.version == version) - and wheel.support_py(for_py_version) - ): - wheels.append(wheel) - return sorted(wheels, key=attrgetter("version_tuple", "distribution"), reverse=True) - - class Version: #: the version bundled with virtualenv - bundle = "bundle" - embed = "embed" + bundle: Final[str] = "bundle" + embed: Final[str] = "embed" #: custom version handlers - non_version = (bundle, embed) + non_version: Final[tuple[str, str]] = (bundle, embed) @staticmethod - def of_version(value): - return None if value in Version.non_version else value + def as_pip_req(distribution: str, version: str | None) -> str: + return f"{distribution}{Version.as_version_spec(version)}" @staticmethod - def as_pip_req(distribution, version): - return f"{distribution}{Version.as_version_spec(version)}" + def as_version_spec(version: str | None) -> str: + return "" if (of_version := Version.of_version(version)) is None else f"=={of_version}" @staticmethod - def as_version_spec(version): - of_version = Version.of_version(version) - return "" if of_version is None else f"=={of_version}" + def of_version(value: str | None) -> str | None: + return None if value in Version.non_version else value __all__ = [ diff --git a/src/virtualenv/util/error.py b/src/virtualenv/util/error.py index a317ddc18..d58a989ac 100644 --- a/src/virtualenv/util/error.py +++ b/src/virtualenv/util/error.py @@ -6,7 +6,7 @@ class ProcessCallFailedError(RuntimeError): """Failed a process call.""" - def __init__(self, code, out, err, cmd) -> None: + def __init__(self, code: int, out: str, err: str, cmd: list[str]) -> None: super().__init__(code, out, err, cmd) self.code = code self.out = out diff --git a/src/virtualenv/util/lock.py b/src/virtualenv/util/lock.py index b250e032f..b3a795865 100644 --- a/src/virtualenv/util/lock.py +++ b/src/virtualenv/util/lock.py @@ -8,24 +8,32 @@ from contextlib import contextmanager, suppress from pathlib import Path from threading import Lock, RLock +from typing import TYPE_CHECKING from filelock import FileLock, Timeout +if TYPE_CHECKING: + from collections.abc import Generator + from types import TracebackType + LOGGER = logging.getLogger(__name__) class _CountedFileLock(FileLock): - def __init__(self, lock_file) -> None: + def __init__(self, lock_file: str) -> None: parent = os.path.dirname(lock_file) - if not os.path.isdir(parent): - with suppress(OSError): - os.makedirs(parent) + with suppress(OSError): + os.makedirs(parent, exist_ok=True) super().__init__(lock_file) self.count = 0 self.thread_safe = RLock() - def acquire(self, timeout=None, poll_interval=0.05): + def acquire( # ty: ignore[invalid-method-override] + self, + timeout: float | None = None, + poll_interval: float = 0.05, + ) -> None: if not self.thread_safe.acquire(timeout=-1 if timeout is None else timeout): raise Timeout(self.lock_file) if self.count == 0: @@ -36,7 +44,7 @@ def acquire(self, timeout=None, poll_interval=0.05): raise self.count += 1 - def release(self, force=False): # noqa: FBT002 + def release(self, force: bool = False) -> None: # ruff:ignore[boolean-default-value-positional-argument] with self.thread_safe: if self.count > 0: if self.count == 1: @@ -52,41 +60,43 @@ def release(self, force=False): # noqa: FBT002 class PathLockBase(ABC): - def __init__(self, folder) -> None: + def __init__(self, folder: str | Path) -> None: path = Path(folder) self.path = path.resolve() if path.exists() else path def __repr__(self) -> str: return f"{self.__class__.__name__}({self.path})" - def __truediv__(self, other): + def __truediv__(self, other: str) -> PathLockBase: return type(self)(self.path / other) @abstractmethod - def __enter__(self): + def __enter__(self) -> None: raise NotImplementedError @abstractmethod - def __exit__(self, exc_type, exc_val, exc_tb): + def __exit__( + self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None + ) -> None: raise NotImplementedError @abstractmethod @contextmanager - def lock_for_key(self, name, no_block=False): # noqa: FBT002 + def lock_for_key(self, name: str, no_block: bool = False) -> Generator[None, None, None]: # ruff:ignore[boolean-default-value-positional-argument] raise NotImplementedError @abstractmethod @contextmanager - def non_reentrant_lock_for_key(self, name): + def non_reentrant_lock_for_key(self, name: str) -> Generator[None, None, None]: raise NotImplementedError class ReentrantFileLock(PathLockBase): - def __init__(self, folder) -> None: + def __init__(self, folder: str | Path) -> None: super().__init__(folder) self._lock = None - def _create_lock(self, name=""): + def _create_lock(self, name: str = "") -> _CountedFileLock: lock_file = str(self.path / f"{name}.lock") with _store_lock: if lock_file not in _lock_store: @@ -94,7 +104,7 @@ def _create_lock(self, name=""): return _lock_store[lock_file] @staticmethod - def _del_lock(lock): + def _del_lock(lock: _CountedFileLock | None) -> None: if lock is not None: with _store_lock, lock.thread_safe: if lock.count == 0: @@ -103,21 +113,23 @@ def _del_lock(lock): def __del__(self) -> None: self._del_lock(self._lock) - def __enter__(self): + def __enter__(self) -> None: self._lock = self._create_lock() self._lock_file(self._lock) - def __exit__(self, exc_type, exc_val, exc_tb): - self._release(self._lock) + def __exit__( + self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None + ) -> None: + self._release(self._lock) # ty: ignore[invalid-argument-type] self._del_lock(self._lock) self._lock = None - def _lock_file(self, lock, no_block=False): # noqa: FBT002 + def _lock_file(self, lock: _CountedFileLock, no_block: bool = False) -> None: # ruff:ignore[boolean-default-value-positional-argument] # multiple processes might be trying to get a first lock... so we cannot check if this directory exist without # a lock, but that lock might then become expensive, and it's not clear where that lock should live. # Instead here we just ignore if we fail to create the directory. with suppress(OSError): - os.makedirs(str(self.path)) + os.makedirs(str(self.path), exist_ok=True) try: lock.acquire(0.0001) @@ -129,41 +141,48 @@ def _lock_file(self, lock, no_block=False): # noqa: FBT002 lock.acquire() @staticmethod - def _release(lock): + def _release(lock: _CountedFileLock) -> None: lock.release() @contextmanager - def lock_for_key(self, name, no_block=False): # noqa: FBT002 + def lock_for_key(self, name: str, no_block: bool = False) -> Generator[None, None, None]: # ruff:ignore[boolean-default-value-positional-argument] lock = self._create_lock(name) try: - try: - self._lock_file(lock, no_block) + with self._lock_and_yield(lock, no_block): yield - finally: - self._release(lock) finally: self._del_lock(lock) lock = None @contextmanager - def non_reentrant_lock_for_key(self, name): + def _lock_and_yield(self, lock: _CountedFileLock, no_block: bool) -> Generator[None, None, None]: + self._lock_file(lock, no_block) + try: + yield + finally: + self._release(lock) + + @contextmanager + def non_reentrant_lock_for_key(self, name: str) -> Generator[None, None, None]: with _CountedFileLock(str(self.path / f"{name}.lock")): yield class NoOpFileLock(PathLockBase): - def __enter__(self): + def __enter__(self) -> None: raise NotImplementedError - def __exit__(self, exc_type, exc_val, exc_tb): + def __exit__( + self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None + ) -> None: raise NotImplementedError @contextmanager - def lock_for_key(self, name, no_block=False): # noqa: ARG002, FBT002 + def lock_for_key(self, name: str, no_block: bool = False) -> Generator[None, None, None]: # ruff:ignore[unused-method-argument, boolean-default-value-positional-argument] yield @contextmanager - def non_reentrant_lock_for_key(self, name): # noqa: ARG002 + def non_reentrant_lock_for_key(self, name: str) -> Generator[None, None, None]: # ruff:ignore[unused-method-argument] yield diff --git a/src/virtualenv/util/path/_permission.py b/src/virtualenv/util/path/_permission.py index 8dcad0ce9..70bb24b6e 100644 --- a/src/virtualenv/util/path/_permission.py +++ b/src/virtualenv/util/path/_permission.py @@ -2,9 +2,13 @@ import os from stat import S_IXGRP, S_IXOTH, S_IXUSR +from typing import TYPE_CHECKING +if TYPE_CHECKING: + from pathlib import Path -def make_exe(filename): + +def make_exe(filename: Path) -> None: original_mode = filename.stat().st_mode levels = [S_IXUSR, S_IXGRP, S_IXOTH] for at in range(len(levels), 0, -1): @@ -18,7 +22,7 @@ def make_exe(filename): continue -def set_tree(folder, stat): +def set_tree(folder: Path, stat: int) -> None: for root, _, files in os.walk(str(folder)): for filename in files: os.chmod(os.path.join(root, filename), stat) diff --git a/src/virtualenv/util/path/_sync.py b/src/virtualenv/util/path/_sync.py index 4dd5a9860..240150a38 100644 --- a/src/virtualenv/util/path/_sync.py +++ b/src/virtualenv/util/path/_sync.py @@ -5,21 +5,29 @@ import shutil import sys from stat import S_IWUSR +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Callable + from pathlib import Path + from types import TracebackType + + _ExcInfo = BaseException | tuple[type[BaseException], BaseException, TracebackType] LOGGER = logging.getLogger(__name__) -def ensure_dir(path): +def ensure_dir(path: Path) -> None: if not path.exists(): LOGGER.debug("create folder %s", path) os.makedirs(str(path)) -def ensure_safe_to_do(src, dest): +def ensure_safe_to_do(src: Path, dest: Path) -> None: if src == dest: msg = f"source and destination is the same {src}" raise ValueError(msg) - if not dest.exists(): + if not dest.exists() and not dest.is_symlink(): return if dest.is_dir() and not dest.is_symlink(): LOGGER.debug("remove directory %s", dest) @@ -29,13 +37,13 @@ def ensure_safe_to_do(src, dest): dest.unlink() -def symlink(src, dest): +def symlink(src: Path, dest: Path) -> None: ensure_safe_to_do(src, dest) LOGGER.debug("symlink %s", _Debug(src, dest)) dest.symlink_to(src, target_is_directory=src.is_dir()) -def copy(src, dest): +def copy(src: Path, dest: Path) -> None: ensure_safe_to_do(src, dest) is_dir = src.is_dir() method = copytree if is_dir else shutil.copy @@ -43,7 +51,7 @@ def copy(src, dest): method(str(src), str(dest)) -def copytree(src, dest): +def copytree(src: str, dest: str) -> None: for root, _, files in os.walk(src): dest_dir = os.path.join(dest, os.path.relpath(root, src)) if not os.path.isdir(dest_dir): @@ -54,20 +62,26 @@ def copytree(src, dest): shutil.copy(src_f, dest_f) -def safe_delete(dest): - def onerror(func, path, exc_info): # noqa: ARG001 - if not os.access(path, os.W_OK): - os.chmod(path, S_IWUSR) - func(path) - else: - raise # noqa: PLE0704 - - kwargs = {"onexc" if sys.version_info >= (3, 12) else "onerror": onerror} - shutil.rmtree(str(dest), ignore_errors=True, **kwargs) +def safe_delete(dest: Path) -> None: + def onerror(func: Callable[[str], object], path: str, exc_info: _ExcInfo) -> None: + exc = exc_info[1] if isinstance(exc_info, tuple) else exc_info + if isinstance(exc, FileNotFoundError): + return + # rmtree also reports os.open/os.close/os.scandir/os.lstat; retrying one of those with a lone path raises + # TypeError over the real error, or deletes nothing while telling rmtree we handled the failure + if func not in {os.unlink, os.rmdir} or os.access(path, os.W_OK): + raise # ruff:ignore[misplaced-bare-raise] + os.chmod(path, os.lstat(path).st_mode | S_IWUSR) + func(path) + + if sys.version_info >= (3, 12): + shutil.rmtree(str(dest), onexc=onerror) + else: + shutil.rmtree(str(dest), onerror=onerror) class _Debug: - def __init__(self, src, dest) -> None: + def __init__(self, src: Path, dest: Path) -> None: self.src = src self.dest = dest @@ -81,5 +95,4 @@ def __str__(self) -> str: "ensure_dir", "safe_delete", "symlink", - "symlink", ] diff --git a/src/virtualenv/util/path/_win.py b/src/virtualenv/util/path/_win.py index 6404cda64..f6a5cc851 100644 --- a/src/virtualenv/util/path/_win.py +++ b/src/virtualenv/util/path/_win.py @@ -1,12 +1,12 @@ from __future__ import annotations -def get_short_path_name(long_name): +def get_short_path_name(long_name: str) -> str: """Gets the short path name of a given long path - http://stackoverflow.com/a/23598461/200291.""" - import ctypes # noqa: PLC0415 - from ctypes import wintypes # noqa: PLC0415 + import ctypes # ruff:ignore[import-outside-top-level] + from ctypes import wintypes # ruff:ignore[import-outside-top-level] - GetShortPathNameW = ctypes.windll.kernel32.GetShortPathNameW # noqa: N806 + GetShortPathNameW = ctypes.windll.kernel32.GetShortPathNameW # ruff:ignore[non-lowercase-variable-in-function] # ty: ignore[unresolved-attribute] GetShortPathNameW.argtypes = [wintypes.LPCWSTR, wintypes.LPWSTR, wintypes.DWORD] GetShortPathNameW.restype = wintypes.DWORD output_buf_size = 0 diff --git a/src/virtualenv/util/subprocess/__init__.py b/src/virtualenv/util/subprocess/__init__.py index e6d5fc885..a7397c6fd 100644 --- a/src/virtualenv/util/subprocess/__init__.py +++ b/src/virtualenv/util/subprocess/__init__.py @@ -1,11 +1,28 @@ from __future__ import annotations import subprocess +from shlex import quote +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Mapping CREATE_NO_WINDOW = 0x80000000 -def run_cmd(cmd): +class LogCmd: + def __init__(self, cmd: list[str], env: Mapping[str, str] | None = None) -> None: + self.cmd = cmd + self.env = env + + def __repr__(self) -> str: + cmd_repr = " ".join(quote(str(c)) for c in self.cmd) + if self.env is not None: + cmd_repr = f"{cmd_repr} env of {self.env!r}" + return cmd_repr + + +def run_cmd(cmd: list[str]) -> tuple[int, str, str]: try: process = subprocess.Popen( cmd, @@ -19,12 +36,13 @@ def run_cmd(cmd): code = process.returncode except OSError as error: code, out, err = error.errno, "", error.strerror - if code == 2 and "file" in err: # noqa: PLR2004 + if code == 2 and err is not None and "file" in err: # ruff:ignore[magic-value-comparison] err = str(error) # FileNotFoundError in Python >= 3.3 - return code, out, err + return code, out, err # ty: ignore[invalid-return-type] __all__ = ( "CREATE_NO_WINDOW", + "LogCmd", "run_cmd", ) diff --git a/src/virtualenv/util/text.py b/src/virtualenv/util/text.py new file mode 100644 index 000000000..1f1b15256 --- /dev/null +++ b/src/virtualenv/util/text.py @@ -0,0 +1,33 @@ +"""Shared text-safety helpers for values written into line-based formats.""" + +from __future__ import annotations + +from typing import Final + +# A line-based format with no escape syntax reads any character str.splitlines() treats as a +# boundary back as an extra line - or, for a script, runs it as one. This is the exact boundary +# set splitlines() recognizes, per the Python string documentation. +_LINE_BOUNDARIES: Final[tuple[str, ...]] = ( + "\n", + "\r", + "\v", + "\f", + "\x1c", + "\x1d", + "\x1e", + "\x85", + chr(0x2028), # line separator + chr(0x2029), # paragraph separator +) + + +def collapse_line_boundaries(text: str) -> str: + """Replace every line-boundary character with a space so text can never become extra lines.""" + for boundary in _LINE_BOUNDARIES: + text = text.replace(boundary, " ") + return text + + +__all__ = [ + "collapse_line_boundaries", +] diff --git a/src/virtualenv/util/zipapp.py b/src/virtualenv/util/zipapp.py index 183dd07db..c5f94ec67 100644 --- a/src/virtualenv/util/zipapp.py +++ b/src/virtualenv/util/zipapp.py @@ -1,21 +1,21 @@ from __future__ import annotations import logging -import os import zipfile +from pathlib import Path -from virtualenv.info import IS_WIN, ROOT +from virtualenv.info import ROOT LOGGER = logging.getLogger(__name__) -def read(full_path): +def read(full_path: str | Path) -> str: sub_file = _get_path_within_zip(full_path) with zipfile.ZipFile(ROOT, "r") as zip_file, zip_file.open(sub_file) as file_handler: return file_handler.read().decode("utf-8") -def extract(full_path, dest): +def extract(full_path: str | Path, dest: Path) -> None: LOGGER.debug("extract %s to %s", full_path, dest) sub_file = _get_path_within_zip(full_path) with zipfile.ZipFile(ROOT, "r") as zip_file: @@ -24,17 +24,19 @@ def extract(full_path, dest): zip_file.extract(info, str(dest.parent)) -def _get_path_within_zip(full_path): - full_path = os.path.realpath(os.path.abspath(str(full_path))) - prefix = f"{ROOT}{os.sep}" - if not full_path.startswith(prefix): - msg = f"full_path={full_path} should start with prefix={prefix}." - raise RuntimeError(msg) - sub_file = full_path[len(prefix) :] - if IS_WIN: - # paths are always UNIX separators, even on Windows, though __file__ still follows platform default - sub_file = sub_file.replace(os.sep, "/") - return sub_file +def _get_path_within_zip(full_path: str | Path) -> str: + # Use Path.relative_to so symlinks and ``..`` segments cannot slip through a string ``startswith`` check. The zipapp + # root is a real file we own so ``resolve`` is safe; anything that does not resolve under ROOT is a bug or an + # attempt to escape the archive and we refuse it. + resolved = Path(full_path).resolve() + root = Path(ROOT).resolve() + try: + relative = resolved.relative_to(root) + except ValueError as exc: + msg = f"full_path={resolved} should be within ROOT={root}" + raise RuntimeError(msg) from exc + # Zip entries always use forward slashes regardless of platform. + return relative.as_posix() __all__ = [ diff --git a/tasks/__main__zipapp.py b/tasks/__main__zipapp.py index 9118e2007..02b865bfe 100644 --- a/tasks/__main__zipapp.py +++ b/tasks/__main__zipapp.py @@ -7,6 +7,13 @@ from functools import cached_property from importlib.abc import SourceLoader from importlib.util import spec_from_file_location +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from collections.abc import Iterator + from importlib.machinery import ModuleSpec + from types import ModuleType, TracebackType + from typing import Self ABS_HERE = os.path.abspath(os.path.dirname(__file__)) @@ -20,7 +27,7 @@ def __init__(self) -> None: self.distributions = self._load("distributions.json") self.__cache = {} - def _load(self, of_file): + def _load(self, of_file: str) -> dict[str, str]: version = ".".join(str(i) for i in sys.version_info[0:2]) per_version = json.loads(self.get_data(of_file).decode()) all_platforms = per_version[version] if version in per_version else per_version["3.9"] @@ -32,22 +39,24 @@ def _load(self, of_file): content.update(all_platforms.get(f"=={sys.platform}", {})) # and finish it off with our platform return content - def __enter__(self): + def __enter__(self) -> Self: return self - def __exit__(self, exc_type, exc_val, exc_tb): + def __exit__( + self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None + ) -> None: self._zip_file.close() - def find_mod(self, fullname): + def find_mod(self, fullname: str) -> str | None: if fullname in self.modules: return self.modules[fullname] return None - def get_filename(self, fullname): + def get_filename(self, fullname: str) -> str | None: zip_path = self.find_mod(fullname) return None if zip_path is None else os.path.join(ABS_HERE, zip_path) - def get_data(self, filename): + def get_data(self, filename: str) -> bytes: if filename.startswith(ABS_HERE): # keep paths relative from the zipfile filename = filename[len(ABS_HERE) + 1 :] @@ -58,17 +67,18 @@ def get_data(self, filename): with self._zip_file.open(filename) as file_handler: return file_handler.read() - def find_distributions(self, context): + def find_distributions(self, context: Any) -> Iterator[Any]: # ruff:ignore[any-type] dist_class = versioned_distribution_class() - name = context.name + if context.name is None: + return + name = context.name.replace("_", "-") if name in self.distributions: - result = dist_class(file_loader=self.get_data, dist_path=self.distributions[name]) - yield result + yield dist_class(file_loader=self.get_data, dist_path=self.distributions[name]) def __repr__(self) -> str: return f"{self.__class__.__name__}(path={ABS_HERE})" - def _register_distutils_finder(self): # noqa: C901 + def _register_distutils_finder(self) -> None: # ruff:ignore[complex-structure] if "distlib" not in self.modules: return @@ -102,14 +112,14 @@ def resources(self) -> list[str]: ] class DistlibFinder: - def __init__(self, path, loader) -> None: + def __init__(self, path: str, loader: Any) -> None: # ruff:ignore[any-type] self.path = path self.loader = loader - def find(self, name): + def find(self, name: str) -> Any: # ruff:ignore[any-type] return Resource(self.path, name, self.loader) - def iterator(self, resource_name): + def iterator(self, resource_name: str) -> Iterator[Any]: resource = self.find(resource_name) if resource is not None: todo = [resource] @@ -125,7 +135,7 @@ def iterator(self, resource_name): else: yield child - from distlib.resources import register_finder # noqa: PLC0415 + from distlib.resources import register_finder # ruff:ignore[import-outside-top-level] zip_file = self._zip_file register_finder(self, lambda module: DistlibFinder(os.path.dirname(module.__file__), self)) @@ -134,20 +144,20 @@ def iterator(self, resource_name): _VER_DISTRIBUTION_CLASS = None -def versioned_distribution_class(): - global _VER_DISTRIBUTION_CLASS # noqa: PLW0603 +def versioned_distribution_class() -> type: + global _VER_DISTRIBUTION_CLASS # ruff:ignore[global-statement] if _VER_DISTRIBUTION_CLASS is None: - from importlib.metadata import Distribution # noqa: PLC0415 + from importlib.metadata import Distribution # ruff:ignore[import-outside-top-level] class VersionedDistribution(Distribution): - def __init__(self, file_loader, dist_path) -> None: + def __init__(self, file_loader: Any, dist_path: str) -> None: # ruff:ignore[any-type] self.file_loader = file_loader self.dist_path = dist_path - def read_text(self, filename): + def read_text(self, filename: str) -> str: return self.file_loader(self.locate_file(filename)).decode("utf-8") - def locate_file(self, path): + def locate_file(self, path: str) -> str: return os.path.join(self.dist_path, path) _VER_DISTRIBUTION_CLASS = VersionedDistribution @@ -155,21 +165,21 @@ def locate_file(self, path): class VersionedFindLoad(VersionPlatformSelect, SourceLoader): - def find_spec(self, fullname, path, target=None): # noqa: ARG002 + def find_spec(self, fullname: str, path: Any, target: ModuleType | None = None) -> ModuleSpec | None: # ruff:ignore[unused-method-argument, any-type] zip_path = self.find_mod(fullname) if zip_path is not None: return spec_from_file_location(name=fullname, loader=self) return None - def module_repr(self, module): + def module_repr(self, module: ModuleType) -> str: raise NotImplementedError -def run(): +def run() -> None: with VersionedFindLoad() as finder: sys.meta_path.insert(0, finder) - finder._register_distutils_finder() # noqa: SLF001 - from virtualenv.__main__ import run as run_virtualenv # noqa: PLC0415, PLC2701 + finder._register_distutils_finder() # ruff:ignore[private-member-access] + from virtualenv.__main__ import run as run_virtualenv # ruff:ignore[import-outside-top-level] run_virtualenv() diff --git a/tasks/apply_upgrade_patch.py b/tasks/apply_upgrade_patch.py new file mode 100644 index 000000000..21e6b81a3 --- /dev/null +++ b/tasks/apply_upgrade_patch.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +import shutil +import subprocess +import sys +from pathlib import PurePosixPath +from typing import Final + + +def main() -> None: + if (git := shutil.which("git")) is None: + msg = "git is required to apply an upgrade patch" + raise SystemExit(msg) + subprocess.run([git, "diff", "--cached", "--exit-code"], check=True) + subprocess.run([git, "apply", "--check", sys.argv[1]], check=True) + subprocess.run([git, "apply", "--cached", sys.argv[1]], check=True) + changes: Final[list[str]] = ( + subprocess + .run( + [git, "diff", "--cached", "--raw", "--no-renames", "-z"], + check=True, + capture_output=True, + encoding="utf-8", + ) + .stdout.rstrip("\0") + .split("\0") + ) + for header, filename in zip(changes[::2], changes[1::2]): + path: Final[PurePosixPath] = PurePosixPath(filename) + allowed: Final[bool] = filename in { + "THIRD-PARTY-NOTICES.md", + "docs/changelog/u.bugfix.rst", + "pylock.zipapp.toml", + "tasks/ci-tools.json", + "tasks/release-requirements.txt", + } or ( + path.parent == PurePosixPath("src/virtualenv/seed/wheels/embed") + and (path.name == "__init__.py" or path.suffix == ".whl") + ) + if not allowed or header.split()[1] not in {"100644", "000000"}: + msg = f"Upgrade patch contains an unsupported path or file mode: {filename}" + raise SystemExit(msg) + subprocess.run([git, "apply", sys.argv[1]], check=True) + + +__all__ = ["main"] + + +if __name__ == "__main__": + main() diff --git a/tasks/check_wheel_age.py b/tasks/check_wheel_age.py new file mode 100644 index 000000000..baeabe16d --- /dev/null +++ b/tasks/check_wheel_age.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +import json +import shutil +import subprocess +import sys +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Final +from urllib.request import urlopen + + +def main() -> None: + cutoff: Final[datetime] = datetime.now(tz=timezone.utc) - timedelta(days=7) + rejected: Final[list[str]] = [] + for wheel in _changed_wheels(): + package, version, *_ = wheel.name.split("-") + with urlopen(f"https://pypi.org/pypi/{package}/{version}/json", timeout=30) as response: + files = json.load(response)["urls"] + if (artifact := next((file for file in files if file["filename"] == wheel.name), None)) is None: + msg = f"PyPI has no upload information for {wheel.name}" + raise SystemExit(msg) + if datetime.fromisoformat(artifact["upload_time_iso_8601"].replace("Z", "+00:00")) > cutoff: + rejected.append(wheel.name) + if rejected: + sys.stderr.write(f"Wheels uploaded less than seven days ago: {', '.join(rejected)}\n") + sys.stdout.write(f"skip={'true' if rejected else 'false'}\n") + + +def _changed_wheels() -> list[Path]: + if (git := shutil.which("git")) is None: + msg = "git is required to find changed wheels" + raise SystemExit(msg) + paths: Final[set[str]] = set() + for arguments in ( + ["diff", "--name-only", "--diff-filter=AM", "-z", "HEAD"], + ["ls-files", "--others", "--exclude-standard", "-z"], + ): + result = subprocess.run( + [git, *arguments, "--", "src/virtualenv/seed/wheels/embed/*.whl"], + check=True, + capture_output=True, + encoding="utf-8", + ) + paths.update(result.stdout.rstrip("\0").split("\0")) + return [Path(path) for path in sorted(paths) if path] + + +__all__ = ["main"] + + +if __name__ == "__main__": + main() diff --git a/tasks/ci-tools.json b/tasks/ci-tools.json new file mode 100644 index 000000000..31401dad6 --- /dev/null +++ b/tasks/ci-tools.json @@ -0,0 +1,35 @@ +{ + "graalpy": { + "assets": {}, + "tag": "graal-25.4.4" + }, + "mermaid": { + "version": "12.0.0" + }, + "nushell": { + "assets": { + "Windows": { + "name": "nu-0.116.0-x86_64-pc-windows-msvc.zip", + "sha256": "745b165d847e34b5dea80c2cfc867383c02b907d288a59a44b25e39bcac14015" + } + }, + "tag": "0.116.0" + }, + "rustpython": { + "assets": { + "Linux": { + "name": "rustpython-release-Linux-x86_64-unknown-linux-gnu", + "sha256": "96b9bf688fde0feb3f392e39e5421382cb7e9c0641d7010c51ced871414b45c6" + }, + "Windows": { + "name": "rustpython-release-Windows-x86_64-pc-windows-msvc.exe", + "sha256": "94a8e11df016e5002b4e4346064813981b50a822a61efe6e6d9310762c46b257" + }, + "macOS": { + "name": "rustpython-release-macOS-aarch64-apple-darwin", + "sha256": "f0376723cdc222dc428df0c1ea771ed638f5c4e9fc6adacf5509fadfb9273bb6" + } + }, + "tag": "2025-03-05-upd-webpack-19" + } +} diff --git a/tasks/cyclonedx_to_spdx.py b/tasks/cyclonedx_to_spdx.py new file mode 100644 index 000000000..55a9645bd --- /dev/null +++ b/tasks/cyclonedx_to_spdx.py @@ -0,0 +1,252 @@ +"""Render the CycloneDX SBOM that hatch_build.py embeds in the wheel as an SPDX 2.3 JSON document.""" + +from __future__ import annotations + +import json +import re +import sys +from datetime import datetime, timezone +from pathlib import Path +from typing import TYPE_CHECKING, Final + +if TYPE_CHECKING: + from collections.abc import Iterator + from typing import NotRequired, TypedDict + + class _Hash(TypedDict): + alg: str + content: str + + class _Reference(TypedDict): + type: str + url: str + + class _Property(TypedDict): + name: str + value: str + + class _Named(TypedDict): + name: str + + class _Licensed(TypedDict): + id: NotRequired[str] + name: NotRequired[str] + + class _License(TypedDict): + expression: NotRequired[str] + license: NotRequired[_Licensed] + + _Component = TypedDict( + "_Component", + { + "type": str, + "bom-ref": str, + "name": str, + "version": NotRequired[str], + "purl": NotRequired[str], + "supplier": NotRequired[_Named], + "hashes": NotRequired[list[_Hash]], + "externalReferences": NotRequired[list[_Reference]], + "copyright": NotRequired[str], + "description": NotRequired[str], + "properties": NotRequired[list[_Property]], + "licenses": NotRequired[list[_License]], + "components": NotRequired[list["_Component"]], + }, + ) + + class _Tools(TypedDict): + components: list[_Component] + + class _Metadata(TypedDict): + timestamp: str + supplier: _Named + component: _Component + tools: _Tools + + class _Dependency(TypedDict): + ref: str + dependsOn: NotRequired[list[str]] + + class CycloneDX(TypedDict): + serialNumber: str + metadata: _Metadata + components: list[_Component] + dependencies: list[_Dependency] + + class _Checksum(TypedDict): + algorithm: str + checksumValue: str + + class _ExternalRef(TypedDict): + referenceCategory: str + referenceType: str + referenceLocator: str + + class _Package(TypedDict): + SPDXID: str + name: str + downloadLocation: str + filesAnalyzed: bool + licenseConcluded: str + licenseDeclared: str + primaryPackagePurpose: str + versionInfo: NotRequired[str] + supplier: NotRequired[str] + checksums: NotRequired[list[_Checksum]] + homepage: NotRequired[str] + copyrightText: NotRequired[str] + summary: NotRequired[str] + comment: NotRequired[str] + externalRefs: NotRequired[list[_ExternalRef]] + licenseComments: NotRequired[str] + + class _Relationship(TypedDict): + spdxElementId: str + relationshipType: str + relatedSpdxElement: str + + class _CreationInfo(TypedDict): + created: str + creators: list[str] + + class Spdx(TypedDict): + spdxVersion: str + dataLicense: str + SPDXID: str + name: str + documentNamespace: str + comment: str + creationInfo: _CreationInfo + packages: list[_Package] + relationships: list[_Relationship] + + +_REPOSITORY: Final[str] = "https://github.com/pypa/virtualenv" +_PURPOSES: Final[dict[str, str]] = { + "application": "APPLICATION", + "library": "LIBRARY", + "platform": "OTHER", +} + + +def main() -> None: + source, target = map(Path, sys.argv[1:3]) + bom: Final[CycloneDX] = json.loads(source.read_text(encoding="utf-8")) + target.write_text(json.dumps(to_spdx(bom), indent=2) + "\n", encoding="utf-8") + + +def to_spdx(bom: CycloneDX) -> Spdx: + root: Final[_Component] = bom["metadata"]["component"] + tools: Final[list[_Component]] = bom["metadata"]["tools"]["components"] + name: Final[str] = f"{root['name']}-{root['version']}" + components: Final[list[_Component]] = list(_walk([root, *bom["components"], *tools])) + return { + "spdxVersion": "SPDX-2.3", + "dataLicense": "CC0-1.0", + "SPDXID": "SPDXRef-DOCUMENT", + "name": name, + # the CycloneDX serial is derived from its content, so the namespace stays stable across rebuilds + "documentNamespace": f"{_REPOSITORY}/sboms/{name}-{bom['serialNumber'].removeprefix('urn:uuid:')}", + "comment": f"Rendered from the CycloneDX SBOM {bom['serialNumber']} (virtualenv.cdx.json), which also carries" + " per-file hashes and the build formulation.", + "creationInfo": { + "created": datetime + .fromisoformat(bom["metadata"]["timestamp"]) + .astimezone(timezone.utc) + .strftime("%Y-%m-%dT%H:%M:%SZ"), + "creators": [ + f"Organization: {bom['metadata']['supplier']['name']}", + *(f"Tool: {tool['name']}-{tool['version']}" for tool in tools if tool["type"] == "application"), + f"Tool: {Path(__file__).name}-{root['version']}", + ], + }, + "packages": [_package(component) for component in components], + "relationships": [ + _relationship("SPDXRef-DOCUMENT", "DESCRIBES", _spdx_id(root["bom-ref"])), + *( + _relationship(_spdx_id(parent["bom-ref"]), "CONTAINS", _spdx_id(child["bom-ref"])) + for parent in components + for child in parent.get("components", []) + if child["type"] != "file" + ), + *( + _relationship(_spdx_id(entry["ref"]), "DEPENDS_ON", _spdx_id(dependency)) + for entry in bom["dependencies"] + for dependency in entry.get("dependsOn", []) + ), + *(_relationship(_spdx_id(tool["bom-ref"]), "BUILD_TOOL_OF", _spdx_id(root["bom-ref"])) for tool in tools), + ], + } + + +def _walk(components: list[_Component]) -> Iterator[_Component]: + # file components mirror the wheels' RECORD entries, which carry no SHA-1 and so cannot become SPDX files + for component in components: + if component["type"] != "file": + yield component + yield from _walk(component.get("components", [])) + + +def _package(component: _Component) -> _Package: + # hatch_build.py emits either one SPDX expression, or the free-form names from License/classifier metadata plus the + # SPDX ids it concluded from them + licenses: Final[list[_License]] = component.get("licenses", []) + concluded: Final[list[str]] = [entry["license"]["id"] for entry in licenses if "id" in entry.get("license", ())] + package: Final[_Package] = { + "SPDXID": _spdx_id(component["bom-ref"]), + "name": component["name"], + "downloadLocation": "NOASSERTION", + "filesAnalyzed": False, + "licenseConcluded": " AND ".join(concluded) or "NOASSERTION", + "licenseDeclared": next((entry["expression"] for entry in licenses if "expression" in entry), "NOASSERTION"), + "primaryPackagePurpose": _PURPOSES[component["type"]], + } + if version := component.get("version"): + package["versionInfo"] = version + if supplier := component.get("supplier"): + package["supplier"] = f"Organization: {supplier['name']}" + if hashes := component.get("hashes"): + package["checksums"] = [ + {"algorithm": entry["alg"].replace("-", ""), "checksumValue": entry["content"]} for entry in hashes + ] + references: Final[list[_Reference]] = component.get("externalReferences", []) + if homepage := next((reference["url"] for reference in references if reference["type"] == "website"), None): + package["homepage"] = homepage + if copyright_text := component.get("copyright"): + package["copyrightText"] = copyright_text + if description := component.get("description"): + package["summary"] = description + if comment := "\n".join( + f"{prop['name']}: {prop['value']}" + for prop in component.get("properties", []) + if prop["name"].startswith("virtualenv:") + ): + package["comment"] = comment + if purl := component.get("purl"): + package["externalRefs"] = [ + {"referenceCategory": "PACKAGE-MANAGER", "referenceType": "purl", "referenceLocator": purl} + ] + if package["licenseDeclared"] == "NOASSERTION" and ( + names := [entry["license"]["name"] for entry in licenses if "name" in entry.get("license", ())] + ): + package["licenseComments"] = f"Declared in package metadata as: {'; '.join(names)}" + return package + + +def _relationship(element: str, kind: str, related: str) -> _Relationship: + return {"spdxElementId": element, "relationshipType": kind, "relatedSpdxElement": related} + + +def _spdx_id(bom_ref: str) -> str: + return f"SPDXRef-{re.sub(r'[^A-Za-z0-9.]+', '-', bom_ref).strip('-')}" + + +__all__ = [ + "CycloneDX", + "to_spdx", +] + + +if __name__ == "__main__": + main() diff --git a/tasks/fuzz_nushell_quote.py b/tasks/fuzz_nushell_quote.py new file mode 100644 index 000000000..87ca99b9e --- /dev/null +++ b/tasks/fuzz_nushell_quote.py @@ -0,0 +1,37 @@ +"""Fuzz NushellActivator.quote for crashes and correct raw-string escaping. + +Run directly: python tasks/fuzz_nushell_quote.py -runs=100000 + +""" + +from __future__ import annotations + +import sys + +import atheris +from fuzz_shared import consume_text + +with atheris.instrument_imports(include=["virtualenv"]): + from virtualenv.activation.nushell import NushellActivator + + +@atheris.instrument_func +def test_one_input(data: bytes) -> None: + string = consume_text(data) + quoted = NushellActivator.quote(string) + wrapping = "#" * (_longest_run(string, "#") + 1) + expected = f"r{wrapping}'{string}'{wrapping}" + assert quoted == expected, (quoted, expected) # ruff:ignore[assert] atheris treats AssertionError as the crash signal + + +def _longest_run(string: str, char: str) -> int: + current = longest = 0 + for c in string: + current = current + 1 if c == char else 0 + longest = max(longest, current) + return longest + + +if __name__ == "__main__": + atheris.Setup(sys.argv, test_one_input) + atheris.Fuzz() diff --git a/tasks/fuzz_powershell_quote.py b/tasks/fuzz_powershell_quote.py new file mode 100644 index 000000000..4e1fd6ee8 --- /dev/null +++ b/tasks/fuzz_powershell_quote.py @@ -0,0 +1,32 @@ +"""Fuzz PowerShellActivator.quote for crashes and correct single-quote escaping. + +Run directly: python tasks/fuzz_powershell_quote.py -runs=100000 + +""" + +from __future__ import annotations + +import sys +from typing import Final + +import atheris +from fuzz_shared import consume_text + +with atheris.instrument_imports(include=["virtualenv"]): + from virtualenv.activation.powershell import PowerShellActivator + +# PowerShell closes a single-quoted string on the ASCII apostrophe and on each curly single quote U+2018-U+201B +_DELIMITERS: Final[frozenset[str]] = frozenset("'\u2018\u2019\u201a\u201b") + + +@atheris.instrument_func +def test_one_input(data: bytes) -> None: + string = consume_text(data) + quoted = PowerShellActivator.quote(string) + expected = "'" + "".join(char * 2 if char in _DELIMITERS else char for char in string) + "'" + assert quoted == expected, (quoted, expected) # ruff:ignore[assert] atheris treats AssertionError as the crash signal + + +if __name__ == "__main__": + atheris.Setup(sys.argv, test_one_input) + atheris.Fuzz() diff --git a/tasks/fuzz_pyenv_cfg.py b/tasks/fuzz_pyenv_cfg.py new file mode 100644 index 000000000..b43bd8687 --- /dev/null +++ b/tasks/fuzz_pyenv_cfg.py @@ -0,0 +1,42 @@ +"""Fuzz the pyvenv.cfg writer and reader for configuration injection. + +This asserts the same invariant as tests/property/test_pyenv_cfg.py, with a coverage-guided driver rather than +hypothesis: a value must never be read back as an extra configuration key. That is the property that #3247 violated, +where a prompt containing a line boundary could override ``home``. + +Run directly: python tasks/fuzz_pyenv_cfg.py -runs=100000 + +""" + +from __future__ import annotations + +import sys +import tempfile +from collections import OrderedDict +from pathlib import Path + +import atheris + +with atheris.instrument_imports(include=["virtualenv"]): + from virtualenv.create.pyenv_cfg import PyEnvCfg + + +@atheris.instrument_func +def test_one_input(data: bytes) -> None: + provider = atheris.FuzzedDataProvider(data) + keys = ("home", "prompt", "version", "base-prefix") + content = OrderedDict( + (keys[i], provider.ConsumeUnicodeNoSurrogates(provider.remaining_bytes() // (len(keys) - i or 1))) + for i in range(len(keys)) + ) + with tempfile.TemporaryDirectory() as folder: + path = Path(folder) / "pyvenv.cfg" + PyEnvCfg(content, path).write() + read_back = PyEnvCfg.from_file(path).content + extra = set(read_back) - set(content) + assert not extra, (extra, dict(content)) # ruff:ignore[assert] atheris treats AssertionError as the crash signal + + +if __name__ == "__main__": + atheris.Setup(sys.argv, test_one_input) + atheris.Fuzz() diff --git a/tasks/fuzz_shared.py b/tasks/fuzz_shared.py new file mode 100644 index 000000000..ca3c6b6cc --- /dev/null +++ b/tasks/fuzz_shared.py @@ -0,0 +1,10 @@ +"""Shared helpers for the fuzz targets in this directory.""" + +from __future__ import annotations + +import atheris + + +def consume_text(data: bytes) -> str: + provider = atheris.FuzzedDataProvider(data) + return provider.ConsumeUnicodeNoSurrogates(provider.remaining_bytes()) diff --git a/tasks/license_policy.py b/tasks/license_policy.py new file mode 100644 index 000000000..90b5db3c2 --- /dev/null +++ b/tasks/license_policy.py @@ -0,0 +1,142 @@ +"""Judge the licenses a virtualenv wheel or zipapp SBOM declares for what the artifact ships.""" + +from __future__ import annotations + +import re +from typing import TYPE_CHECKING, Final + +if TYPE_CHECKING: + from collections.abc import Iterator + from typing import NotRequired, TypedDict + + class _Licensed(TypedDict): + id: NotRequired[str] + name: NotRequired[str] + + class _License(TypedDict): + expression: NotRequired[str] + license: NotRequired[_Licensed] + + _Component = TypedDict( + "_Component", + { + "type": str, + "bom-ref": str, + "hashes": NotRequired[list[object]], + "licenses": NotRequired[list[_License]], + "components": NotRequired[list["_Component"]], + }, + ) + +# the licensing policy in docs/development.rst refers to this set +ALLOWED_LICENSES: Final[frozenset[str]] = frozenset({ + "Apache-2.0", + "BSD-2-Clause", + "BSD-3-Clause", + "ISC", + "MIT", + "PSF-2.0", + "Unlicense", +}) +# SPDX matches license ids without regard to case, and operators only in upper case +_ALLOWED_KEYS: Final[frozenset[str]] = frozenset(identifier.casefold() for identifier in ALLOWED_LICENSES) +_OPERATORS: Final[frozenset[str]] = frozenset({"AND", "OR", "WITH", "(", ")"}) +_TOKEN: Final[re.Pattern[str]] = re.compile(r"[()]|[^\s()]+") + + +def license_problems(components: list[_Component]) -> list[str]: + return [ + f"{component['bom-ref']}: {problem}" for component in _shipped(components) if (problem := _judge(component)) + ] + + +def _shipped(components: list[_Component]) -> Iterator[_Component]: + for component in components: + # the SBOM gives bytes only to what the artifact carries; a declared requirement resolves at install time, and a + # library a seed wheel vendors ships under the terms of that wheel + if component["type"] == "library" and ( + "hashes" in component or any(child["type"] == "file" for child in component.get("components", [])) + ): + yield component + yield from _shipped(component.get("components", [])) + + +def _judge(component: _Component) -> str | None: + licenses: Final[list[_License]] = component.get("licenses", []) + terms: Final[list[str]] = [ + entry["expression"] if "expression" in entry else entry["license"]["id"] + for entry in licenses + if "expression" in entry or "id" in entry["license"] + ] + if not terms: + if names := [entry["license"]["name"] for entry in licenses]: + return f"declares license names {names} with no SPDX id; map them in hatch_build.py _CLASSIFIER_LICENSES" + return "declares no license" + try: + denied: Final[list[str]] = [term for term in terms if not _allowed(term)] + except ValueError as error: + return str(error) + if denied: + return f"license {' AND '.join(denied)} is outside ALLOWED_LICENSES {sorted(ALLOWED_LICENSES)}" + return None + + +def _allowed(expression: str) -> bool: + tokens: Final[list[str]] = _TOKEN.findall(expression)[::-1] + try: + verdict: Final[bool] = _any_of(tokens) + except ValueError as error: + msg = f"license expression {expression!r} is malformed: {error}" + raise ValueError(msg) from error + if tokens: + msg = f"license expression {expression!r} is malformed: unexpected {tokens[-1]!r}" + raise ValueError(msg) + return verdict + + +def _any_of(tokens: list[str]) -> bool: + verdict = _all_of(tokens) + while tokens and tokens[-1] == "OR": + tokens.pop() + verdict |= _all_of(tokens) + return verdict + + +def _all_of(tokens: list[str]) -> bool: + verdict = _term(tokens) + while tokens and tokens[-1] == "AND": + tokens.pop() + verdict &= _term(tokens) + return verdict + + +def _term(tokens: list[str]) -> bool: + if (token := _pop(tokens)) == "(": + verdict: Final[bool] = _any_of(tokens) + if (closing := _pop(tokens)) != ")": + msg = f"expected ')', got {closing!r}" + raise ValueError(msg) + return verdict + if token in _OPERATORS: + msg = f"unexpected {token!r}" + raise ValueError(msg) + # an exception grants extra permissions on top of the license it modifies, so that license alone decides + if tokens and tokens[-1] == "WITH": + tokens.pop() + if (exception := _pop(tokens)) in _OPERATORS: + msg = f"unexpected {exception!r}" + raise ValueError(msg) + return token.casefold() in _ALLOWED_KEYS + + +def _pop(tokens: list[str]) -> str: + if not tokens: + msg: Final[str] = "it ends early" + raise ValueError(msg) + return tokens.pop() + + +__all__ = [ + "ALLOWED_LICENSES", + "license_problems", +] diff --git a/tasks/make_zipapp.py b/tasks/make_zipapp.py index 608efcf8c..0954eed3a 100644 --- a/tasks/make_zipapp.py +++ b/tasks/make_zipapp.py @@ -3,6 +3,7 @@ from __future__ import annotations import argparse +import hashlib import io import json import os @@ -11,31 +12,119 @@ import sys import zipapp import zipfile -from collections import defaultdict, deque -from email import message_from_string +from collections import defaultdict +from dataclasses import dataclass +from datetime import datetime, timezone from pathlib import Path, PurePosixPath -from shlex import quote -from stat import S_IWUSR from tempfile import TemporaryDirectory +from typing import TYPE_CHECKING, Any, Final +from urllib.request import urlopen from packaging.markers import Marker -from packaging.requirements import Requirement + +if sys.version_info >= (3, 11): + import tomllib +else: + import tomli as tomllib + +if TYPE_CHECKING: + from typing import TypedDict + + from typing_extensions import NotRequired + + class LockedWheel(TypedDict): + url: str + hashes: dict[str, str] + + class LockedPackage(TypedDict): + name: str + version: str + marker: NotRequired[str] + wheels: NotRequired[list[LockedWheel]] + HERE = Path(__file__).parent.absolute() -VERSIONS = [f"3.{i}" for i in range(13, 7, -1)] +VERSIONS = [f"3.{i}" for i in range(14, 7, -1)] +LOCK: Final[Path] = HERE.parent / "pylock.zipapp.toml" +PLATFORMS: Final[tuple[str, ...]] = ("darwin", "linux", "win32") -def main(): +def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--dest", default="virtualenv.pyz") + parser.add_argument("--wheel-dir", type=Path, help="bundle the virtualenv wheel from here instead of building one") args = parser.parse_args() with TemporaryDirectory() as folder: - packages = get_wheels_for_support_versions(Path(folder)) + packages = get_wheels_for_support_versions(Path(folder), args.wheel_dir) create_zipapp(os.path.abspath(args.dest), packages) -def create_zipapp(dest, packages): +def get_wheels_for_support_versions( + folder: Path, wheel_dir: Path | None +) -> dict[str, dict[str, dict[str, WheelForVersion]]]: + packages: defaultdict[str, dict[str, dict[str, WheelForVersion]]] = defaultdict(lambda: {"==any": {}}) + wheel = build_virtualenv_wheel(folder) if wheel_dir is None else next(wheel_dir.glob("virtualenv-*.whl")) + packages["virtualenv"]["==any"][wheel.name] = WheelForVersion(wheel, list(VERSIONS)) + for package in tomllib.loads(LOCK.read_text(encoding="utf-8"))["packages"]: + wheel = download_locked_wheel(package, folder) + packages[package["name"]]["==any"][wheel.name] = WheelForVersion(wheel, python_versions_for(package)) + for name, platforms in packages.items(): + for wheel_name, wheel_for_version in platforms["==any"].items(): + sys.stdout.write(f"{name}: {wheel_name} for {' '.join(wheel_for_version.versions)}\n") + return packages + + +def build_virtualenv_wheel(into: Path) -> Path: + with TemporaryDirectory() as temp_folder: + # pip does not guarantee building from the same source tree in parallel is safe, so build from a copy + source = Path(temp_folder) / HERE.parent.name + shutil.copytree( + HERE.parent, source, ignore=shutil.ignore_patterns(".tox", ".tox4", "venv", "__pycache__", "*.pyz") + ) + subprocess.run( + [sys.executable, "-m", "pip", "wheel", "-q", "--no-deps", "-w", str(into), str(source)], check=True + ) + return next(into.glob("virtualenv-*.whl")) + + +def download_locked_wheel(package: LockedPackage, into: Path) -> Path: + # the zipapp cannot load compiled extensions, so only a pure-Python wheel can be bundled + if not (wheels := [wheel for wheel in package.get("wheels", []) if wheel["url"].endswith("-none-any.whl")]): + msg = f"{LOCK.name} has no pure-Python wheel for {package['name']} {package['version']}" + raise RuntimeError(msg) + url, expected = wheels[0]["url"], wheels[0]["hashes"]["sha256"] + with urlopen(url, timeout=60) as response: # ruff:ignore[suspicious-url-open-usage] # lock URL, sha256-checked + content = response.read() + if (actual := hashlib.sha256(content).hexdigest()) != expected: + msg = f"{url} has sha256 {actual}, but {LOCK.name} records {expected}" + raise RuntimeError(msg) + (dest := into / url.rsplit("/", 1)[1]).write_bytes(content) + return dest + + +def python_versions_for(package: LockedPackage) -> list[str]: + if (marker := package.get("marker")) is None: + return list(VERSIONS) + parsed, versions = Marker(marker), [] + for version in VERSIONS: + environment = {"python_version": version, "python_full_version": f"{version}.0"} + # every bundled wheel is keyed as ==any, so a marker that differs per platform would ship the build host's pick + if len(matches := {parsed.evaluate({**environment, "sys_platform": p}) for p in PLATFORMS}) != 1: + msg = f"{LOCK.name} marks {package['name']} as platform specific ({marker}), which the zipapp cannot select" + raise RuntimeError(msg) + if matches.pop(): + versions.append(version) + return versions + + +@dataclass(frozen=True) +class WheelForVersion: + wheel: Path + versions: list[str] + + +def create_zipapp(dest: str, packages: dict[str, dict[str, dict[str, WheelForVersion]]]) -> None: bio = io.BytesIO() base = PurePosixPath("__virtualenv__") modules = defaultdict(lambda: defaultdict(dict)) @@ -43,18 +132,24 @@ def create_zipapp(dest, packages): with zipfile.ZipFile(bio, "w") as zip_app: write_packages_to_zipapp(base, dist, modules, packages, zip_app) modules_json = json.dumps(modules, indent=2) - zip_app.writestr("modules.json", modules_json) + zip_app.writestr(_entry("modules.json"), modules_json) distributions_json = json.dumps(dist, indent=2) - zip_app.writestr("distributions.json", distributions_json) - zip_app.writestr("__main__.py", (HERE / "__main__zipapp.py").read_bytes()) + zip_app.writestr(_entry("distributions.json"), distributions_json) + zip_app.writestr(_entry("__main__.py"), (HERE / "__main__zipapp.py").read_bytes()) bio.seek(0) zipapp.create_archive(bio, dest) - print(f"zipapp created at {dest} with size {os.path.getsize(dest) / 1024 / 1024:.2f}MB") # noqa: T201 + print(f"zipapp created at {dest} with size {os.path.getsize(dest) / 1024 / 1024:.2f}MB") # ruff:ignore[print] -def write_packages_to_zipapp(base, dist, modules, packages, zip_app): # noqa: C901, PLR0912 +def write_packages_to_zipapp( # ruff:ignore[complex-structure, too-many-branches] + base: PurePosixPath, + dist: dict[str, Any], + modules: dict[str, Any], + packages: dict[str, dict[str, dict[str, WheelForVersion]]], + zip_app: zipfile.ZipFile, +) -> None: has = set() - for name, p_w_v in packages.items(): # noqa: PLR1702 + for name, p_w_v in packages.items(): # ruff:ignore[too-many-nested-blocks] for platform, w_v in p_w_v.items(): for wheel_data in w_v.values(): wheel = wheel_data.wheel @@ -71,218 +166,31 @@ def write_packages_to_zipapp(base, dist, modules, packages, zip_app): # noqa: C for version in wheel_data.versions: modules[version][platform][key] = str(dest) if dest.parent.suffix == ".dist-info": + dist_name = dest.parent.stem.split("-")[0].replace("_", "-") for version in wheel_data.versions: - dist[version][platform][dest.parent.stem.split("-")[0]] = str(dest.parent) + dist[version][platform][dist_name] = str(dest.parent) dest_str = str(dest) if dest_str in has: continue has.add(dest_str) if "/tests/" in dest_str or "/docs/" in dest_str: continue - print(dest_str) # noqa: T201 + print(dest_str) # ruff:ignore[print] content = wheel_zip.read(filename) - zip_app.writestr(dest_str, content) + zip_app.writestr(_entry(dest_str), content) del content -class WheelDownloader: - def __init__(self, into) -> None: - if into.exists(): - shutil.rmtree(into) - into.mkdir(parents=True) - self.into = into - self.collected = defaultdict(lambda: defaultdict(dict)) - self.pip_cmd = [str(Path(sys.executable).parent / "pip")] - self._cmd = [*self.pip_cmd, "download", "-q", "--no-deps", "--no-cache-dir", "--dest", str(self.into)] - - def run(self, target, versions): - whl = self.build_sdist(target) - todo = deque((version, None, whl) for version in versions) - wheel_store = {} - while todo: - version, platform, dep = todo.popleft() - dep_str = dep.name.split("-")[0] if isinstance(dep, Path) else dep.name - if dep_str in self.collected[version] and platform in self.collected[version][dep_str]: - continue - whl = self._get_wheel(dep, platform[2:] if platform and platform.startswith("==") else None, version) - if whl is None: - if dep_str not in wheel_store: - msg = f"failed to get {dep_str}, have {wheel_store}" - raise RuntimeError(msg) - whl = wheel_store[dep_str] - else: - wheel_store[dep_str] = whl - self.collected[version][dep_str][platform] = whl - todo.extend(self.get_dependencies(whl, version)) - - def _get_wheel(self, dep, platform, version): - if isinstance(dep, Requirement): - before = set(self.into.iterdir()) - if self._download( - platform, - False, # noqa: FBT003 - "--python-version", - version, - "--only-binary", - ":all:", - str(dep), - ): - self._download(platform, True, "--python-version", version, str(dep)) # noqa: FBT003 - after = set(self.into.iterdir()) - new_files = after - before - assert len(new_files) <= 1 # noqa: S101 - if not len(new_files): - return None - new_file = next(iter(new_files)) - if new_file.suffix == ".whl": - return new_file - dep = new_file - new_file = self.build_sdist(dep) - assert new_file.suffix == ".whl" # noqa: S101 - return new_file - - def _download(self, platform, stop_print_on_fail, *args): - exe_cmd = self._cmd + list(args) - if platform is not None: - exe_cmd.extend(["--platform", platform]) - return run_suppress_output(exe_cmd, stop_print_on_fail=stop_print_on_fail) - - @staticmethod - def get_dependencies(whl, version): - with zipfile.ZipFile(str(whl), "r") as zip_file: - name = "/".join([f"{'-'.join(whl.name.split('-')[0:2])}.dist-info", "METADATA"]) - with zip_file.open(name) as file_handler: - metadata = message_from_string(file_handler.read().decode("utf-8")) - deps = metadata.get_all("Requires-Dist") - if deps is None: - return - for dep in deps: - req = Requirement(dep) - markers = getattr(req.marker, "_markers", ()) or () - if any( - m - for m in markers - if isinstance(m, tuple) and len(m) == 3 and m[0].value == "extra" # noqa: PLR2004 - ): - continue - py_versions = WheelDownloader._marker_at(markers, "python_version") - if py_versions: - marker = Marker('python_version < "1"') - marker._markers = [ # noqa: SLF001 - markers[ver] for ver in sorted(i for i in set(py_versions) | {i - 1 for i in py_versions} if i >= 0) - ] - matches_python = marker.evaluate({"python_version": version}) - if not matches_python: - continue - deleted = 0 - for ver in py_versions: - deleted += WheelDownloader._del_marker_at(markers, ver - deleted) - platforms = [] - platform_positions = WheelDownloader._marker_at(markers, "sys_platform") - deleted = 0 - for pos in platform_positions: # can only be or meaningfully - platform = f"{markers[pos][1].value}{markers[pos][2].value}" - deleted += WheelDownloader._del_marker_at(markers, pos - deleted) - platforms.append(platform) - if not platforms: - platforms.append(None) - for platform in platforms: - yield version, platform, req - - @staticmethod - def _marker_at(markers, key): - return [ - i - for i, m in enumerate(markers) - if isinstance(m, tuple) and len(m) == 3 and m[0].value == key # noqa: PLR2004 - ] - - @staticmethod - def _del_marker_at(markers, at): - del markers[at] - deleted = 1 - op = max(at - 1, 0) - if markers and isinstance(markers[op], str): - del markers[op] - deleted += 1 - return deleted - - def build_sdist(self, target): - if target.is_dir(): - # pip 20.1 no longer guarantees this to be parallel safe, need to copy/lock - with TemporaryDirectory() as temp_folder: - folder = Path(temp_folder) / target.name - shutil.copytree( - str(target), - str(folder), - ignore=shutil.ignore_patterns(".tox", ".tox4", "venv", "__pycache__", "*.pyz"), - ) - try: - return self._build_sdist(self.into, folder) - finally: - # permission error on Windows <3.7 https://bugs.python.org/issue26660 - def onerror(func, path, exc_info): # noqa: ARG001 - os.chmod(path, S_IWUSR) - func(path) - - shutil.rmtree(str(folder), onerror=onerror) - - else: - return self._build_sdist(target.parent / target.stem, target) - - def _build_sdist(self, folder, target): - if not folder.exists() or not list(folder.iterdir()): - cmd = [*self.pip_cmd, "wheel", "-w", str(folder), "--no-deps", str(target), "-q"] - run_suppress_output(cmd, stop_print_on_fail=True) - return next(iter(folder.iterdir())) - - -def run_suppress_output(cmd, stop_print_on_fail=False): # noqa: FBT002 - process = subprocess.Popen( - cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True, - encoding="utf-8", +def _entry(name: str) -> zipfile.ZipInfo: + # the build time and OS would otherwise end up in each entry header; 1580601600 is hatchling's fallback, so the + # entries share the timestamp of the SBOM appended after them + epoch: Final[int] = int(os.environ.get("SOURCE_DATE_EPOCH", "1580601600")) + entry: Final[zipfile.ZipInfo] = zipfile.ZipInfo( + name, datetime.fromtimestamp(epoch, tz=timezone.utc).timetuple()[:6] ) - out, err = process.communicate() - if stop_print_on_fail and process.returncode != 0: - print(f"exit with {process.returncode} of {' '.join(quote(i) for i in cmd)}", file=sys.stdout) # noqa: T201 - if out: - print(out, file=sys.stdout) # noqa: T201 - if err: - print(err, file=sys.stderr) # noqa: T201 - raise SystemExit(process.returncode) - return process.returncode - - -def get_wheels_for_support_versions(folder): - downloader = WheelDownloader(folder / "wheel-store") - downloader.run(HERE.parent, VERSIONS) - packages = defaultdict(lambda: defaultdict(lambda: defaultdict(WheelForVersion))) - for version, collected in downloader.collected.items(): - for pkg, platform_to_wheel in collected.items(): - name = Requirement(pkg).name - for platform, wheel in platform_to_wheel.items(): - pl = platform or "==any" - wheel_versions = packages[name][pl][wheel.name] - wheel_versions.versions.append(version) - wheel_versions.wheel = wheel - for name, p_w_v in packages.items(): - for platform, w_v in p_w_v.items(): - print(f"{name} - {platform}") # noqa: T201 - for wheel, wheel_versions in w_v.items(): - print(f"{' '.join(wheel_versions.versions)} of {wheel} (use {wheel_versions.wheel})") # noqa: T201 - return packages - - -class WheelForVersion: - def __init__(self, wheel=None, versions=None) -> None: - self.wheel = wheel - self.versions = versions or [] - - def __repr__(self) -> str: - return f"{self.__class__.__name__}({self.wheel!r}, {self.versions!r})" + entry.create_system = 3 + entry.external_attr = 0o644 << 16 + return entry if __name__ == "__main__": diff --git a/tasks/pytest_diagnostics.py b/tasks/pytest_diagnostics.py new file mode 100644 index 000000000..052f56fdd --- /dev/null +++ b/tasks/pytest_diagnostics.py @@ -0,0 +1,109 @@ +"""Keep stack evidence when pytest stalls outside a test's timeout window.""" + +from __future__ import annotations + +import atexit +import faulthandler +import os +import sys +import threading +from datetime import datetime, timezone +from pathlib import Path +from typing import TYPE_CHECKING, Final + +import pytest + +if TYPE_CHECKING: + from typing import TextIO + + +# faulthandler has one timer per process, including nested pytest sessions. +_DIAGNOSTICS: Final[list[_Diagnostics]] = [] +# faulthandler's native timer walks other threads' frames without the GIL and can crash or hang CPython, so it only +# backs up the dump thread: re-armed after every dump, it fires once a thread holds the GIL long enough to stall it +_STALL_INTERVALS: Final[int] = 10 + + +def pytest_addoption(parser: pytest.Parser) -> None: + parser.addoption( + "--diagnostics-dir", + type=Path, + default=Path(".tox/hang-diagnostics"), + help="directory for per-process stack dumps and test events", + ) + parser.addoption("--diagnostics-interval", type=float, default=60, help="seconds between stack dumps") + + +@pytest.hookimpl(trylast=True) +def pytest_configure(config: pytest.Config) -> None: + interval: Final[float] = config.getoption("diagnostics_interval") + if interval <= 0: + msg = "--diagnostics-interval must be positive" + raise pytest.UsageError(msg) + if not _DIAGNOSTICS: + _DIAGNOSTICS.append(_Diagnostics(config.getoption("diagnostics_dir"), interval)) + config.pluginmanager.register(_DIAGNOSTICS[0], "session-diagnostics") + _DIAGNOSTICS[0].retain_crash_traceback() + + +class _Diagnostics: + def __init__(self, directory: Path, interval: float) -> None: + directory.mkdir(parents=True, exist_ok=True) + worker: Final[str] = os.environ.get("PYTEST_XDIST_WORKER", "controller") + self.stream: Final[TextIO] = (directory / f"{worker}-{os.getpid()}.log").open( + "w", encoding="utf-8", buffering=1 + ) + self.interval: Final[float] = interval + # pytest also arms and cancels the native timer, and free-threaded builds race concurrent faulthandler calls + self._faulthandler_lock: Final[threading.Lock] = threading.Lock() + self._stopped: Final[threading.Event] = threading.Event() + self._record(f"START pid={os.getpid()} parent={os.getppid()} python={sys.version}") + self._arm_stall_timer() + self._dumper: Final[threading.Thread] = threading.Thread( + target=self._dump_periodically, name="pytest-diagnostics", daemon=True + ) + self._dumper.start() + atexit.register(self._close) + + def _record(self, message: str) -> None: + self.stream.write(f"{datetime.now(timezone.utc).isoformat()} {message}\n") + + def _arm_stall_timer(self) -> None: + with self._faulthandler_lock: + faulthandler.dump_traceback_later(self.interval * _STALL_INTERVALS, repeat=True, file=self.stream) + + def _dump_periodically(self) -> None: + while not self._stopped.wait(self.interval): + self._record("DUMP") + # holds the GIL, or stops the world on free-threaded builds, so no frame is freed while it reads + faulthandler.dump_traceback(file=self.stream) + self._arm_stall_timer() + + def retain_crash_traceback(self) -> None: + # pytest's captured stderr can disappear with a crashed xdist worker. + with self._faulthandler_lock: + faulthandler.enable(file=self.stream) + + def pytest_runtest_logstart(self, nodeid: str) -> None: + self._record(f"TEST {nodeid}") + + def pytest_runtest_logreport(self, report: pytest.TestReport) -> None: + self._record(f"{report.when} {report.outcome} {report.nodeid}") + + @pytest.hookimpl(trylast=True) + def pytest_unconfigure(self) -> None: + self.retain_crash_traceback() + self._record("STOP") + + def _close(self) -> None: + # Keep dumping through interpreter shutdown, after pytest has returned. + self._stopped.set() + self._dumper.join() + with self._faulthandler_lock: + faulthandler.cancel_dump_traceback_later() + faulthandler.disable() + self._record("EXIT") + self.stream.close() + + +__all__ = ["pytest_addoption", "pytest_configure"] diff --git a/tasks/release-requirements.txt b/tasks/release-requirements.txt new file mode 100644 index 000000000..328938871 --- /dev/null +++ b/tasks/release-requirements.txt @@ -0,0 +1,1567 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile --group pyproject.toml:release --universal --python-version 3.15 --generate-hashes -o tasks/release-requirements.txt +alabaster==1.0.0 \ + --hash=sha256:c00dca57bca26fa62a6d7d0a9fcce65f3e026e9bfe33e9c538fd3fbb2144fd9e \ + --hash=sha256:fc6786402dc3fcb2de3cabd5fe455a2db534b371124f1f21de8731783dec828b + # via sphinx +annotated-types==0.8.0 \ + --hash=sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7 \ + --hash=sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0 + # via pydantic +arrow==1.4.0 \ + --hash=sha256:749f0769958ebdc79c173ff0b0670d59051a535fa26e8eba02953dc19eb43205 \ + --hash=sha256:ed0cc050e98001b8779e84d461b0098c4ac597e88704a655582b21d116e526d7 + # via isoduration +attrs==26.1.0 \ + --hash=sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309 \ + --hash=sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32 + # via + # jsonschema + # referencing +babel==2.18.0 \ + --hash=sha256:b80b99a14bd085fcacfa15c9165f651fbb3406e66cc603abf11c5750937c992d \ + --hash=sha256:e2b422b277c2b9a9630c1d7903c2a00d0830c409c59ac8cae9081c92f1aeba35 + # via sphinx +beartype==0.23.0rc2 \ + --hash=sha256:d589e5c6cf1859e4cf355db52ddcd08e2eca98784d7d73dc5f4491f423f94ee5 \ + --hash=sha256:fffbb5aa86562b034038f2c90da71cac463247374c89981c937685f47a75223c + # via + # virtualenv (pyproject.toml:release) + # spdx-tools +black==26.5.1 \ + --hash=sha256:0e48b87e03bf109288e55cfceadcfa15ff5470aca2851a851950ed2926f450d7 \ + --hash=sha256:1037d5ac7b7b310b2632ad867ec8d0e4c4819dcdb0b820f63135da746a24e418 \ + --hash=sha256:1ef92b76f7733f282fd096ea406200b5a286c42947412b0eaff3a74e3616cefe \ + --hash=sha256:1f7ea64ebfa01b50f693508fc39f875e264446d3b097088f84f203b9d09618a0 \ + --hash=sha256:22f2cd76d069cc54c71f10360744ba8983fbb616903b4304a85b734915c8e1b4 \ + --hash=sha256:2b36cf2ddf5566e205f6535f782a62194a184d33e175b64ae8c40b1737522be3 \ + --hash=sha256:30d3c14661f2792e9142cce3eeeb1cbc175b3eb5f733be0c8eeb99651e52b0c3 \ + --hash=sha256:32d5ea7f6c8bdfa6e648326ebca1f02b0764e2a029edc6f8dce2627e19d468c3 \ + --hash=sha256:3915f256e75a2d7cf88d8953d37f780455dc586cc72dee059c528fe77f581217 \ + --hash=sha256:4ad6fa01f941920f54f2bbb35f3df7673428a0ef98a0b0840c2eaef3b110efa8 \ + --hash=sha256:4ed7f7da04046d2e488437170797d3b4a4ad83906683bcb7dfc68b673bbce5e2 \ + --hash=sha256:5119fa92ae61f786e8c3662fd60aece1d0a2dd5cca5d0c79417a95e7a4272a59 \ + --hash=sha256:577f21094ea469ef92ec1adaf2c9441a226d2144d01a5be2fa823cecf6543e50 \ + --hash=sha256:58b4bd92cf88aacf83d88479c8f9caee044b1ec55f2451a337354a7ea2590a22 \ + --hash=sha256:5c34b25da232ead53a6f335b76dbea124f4d152ad568b9080d6f944bc2b34b52 \ + --hash=sha256:87ed5c6f450580a2f6790bc7cbfb016dfc73bc750249762268a3695361315eef \ + --hash=sha256:89c93167a74d3a75dfaa38a5c7cca015537d5820dd7f17d63267d674a61cae90 \ + --hash=sha256:96ae2c733b2aabdd9986e2c5df628ff3473676cd1c5faded1ff496cf6d74083c \ + --hash=sha256:9942db8888e06943c5dde66ca0037dcff82a2a4ec1ad0ada9e0d2ee9d9823893 \ + --hash=sha256:9d98d4137277c75dfb898ec8d846c4fd68ba1e9cf77f95e2865c203dc18f4c3d \ + --hash=sha256:a1dca32d9f1784af512a13410ec204c6f7f0aa9797a111c42e1c03449821c264 \ + --hash=sha256:dd321f668053961824bcc1be1cc1df748b2d7e4fa28086b08331e577b0100a73 \ + --hash=sha256:e1a26503279b6b310669fb0b219c39e4820b77e8189fe80f522bb511f247db0a \ + --hash=sha256:e88976690a64b0af98312ca958415849cb42423423c5f2ee74af4b49a97a2168 \ + --hash=sha256:ea8d16dc41655aa113cd64665e7219446cd7e4ff2248d7178eaa905190c86b18 \ + --hash=sha256:ecb3e624844c798144e9bd986954e0adc81d8911a1f30f375e1252fe26e8c294 \ + --hash=sha256:ed1a20af114c301a0269bf01163d51dbef72737fd65f850001e7cbe7f3c7abae + # via docstrfmt +boolean-py==5.0 \ + --hash=sha256:60cbc4bad079753721d32649545505362c754e121570ada4658b852a3a318d95 \ + --hash=sha256:ef28a70bd43115208441b53a045d1549e2f0ec6e3d08a9d142cbc41c1938e8d9 + # via license-expression +cachetools==7.2.0 \ + --hash=sha256:3045213f186b89fdd95d94441354c4bd87c570b7a38d8c3fd1d9dd37f6dc90d8 \ + --hash=sha256:bcac1a1b8da6909994a2957238a57b8140dab7c5c5c69a43669654fe87a33c1d + # via tox +certifi==2026.7.22 \ + --hash=sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775 \ + --hash=sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55 + # via requests +cffi==2.1.1 ; platform_python_implementation != 'PyPy' \ + --hash=sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e \ + --hash=sha256:06c72bb76605a4b0cd0aad6930b69d4baf7dd5d806cfc409b824191099700e66 \ + --hash=sha256:0beceaabe56af686895136a2de78db54ecd8e4046b236b8fd6d6cb61389e9bf2 \ + --hash=sha256:154852545011f779917b11c78db2358d095da62a9a172b78ad0a583ee5adc0d0 \ + --hash=sha256:194cffa889098ced9976c3fc6340305e43f6303657d298da55366907c05c22d6 \ + --hash=sha256:19ee6127ee34de7d83ce3d371ebc5ed91addbdcc39f9ab15ce4eb35a4e534971 \ + --hash=sha256:1a18a57b58cfb21fc28d72e876acf10eaed67a1ed96226f92af4df681d571c4c \ + --hash=sha256:1aa5645c30469b09530c4ebca77ebf8f17618293c58f8549cb1a543a50236e7d \ + --hash=sha256:1dea0e4d7d4f11f619fe8c1d76caf49e24405b4b5743c0e3be16a500ecd930c9 \ + --hash=sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517 \ + --hash=sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735 \ + --hash=sha256:246fa40ce8645a614ff682e0b70f37134e460eaf93a775e0cbe3cca585a67a80 \ + --hash=sha256:25792eac27877609e7bb06d42ff88278a6624fff2ba9bbb523c09616b117e80f \ + --hash=sha256:27350daa11d4f10c540e6e89dada4c54feb7256ad03e9a4dc075ebad7ba360d1 \ + --hash=sha256:28907ab9bfb6aa13184cfc17c6b8e1023c5ab6fd7076d8c20a35e59fe04f8f29 \ + --hash=sha256:2ae64be792b8966f2c69538199728b290e34726562896df1e5dc8ffd8d8188e8 \ + --hash=sha256:31348097ff5bbe827ccc41795d4dd099d9f0625e7def00ee653c137a490c2a6c \ + --hash=sha256:3143d81e29e1e20a9ce10901ec369012947876596f75a222235965f2b7ae832e \ + --hash=sha256:3222ba5d678f80a030e6afbcc33dc1ae5cb45facabb61cee2c7016b8432fde48 \ + --hash=sha256:3311ed60d36f83378794e1009ac6258bafbf81f7888b4caa7b35a521e3f95813 \ + --hash=sha256:334644fbac4eff73d985a17a91226df55d0f394160c4cfb880e084c8f7161cac \ + --hash=sha256:34e261f78cb6ceaaa36f42f2613f4380d94d9c759a9c73c769ee6e0247364632 \ + --hash=sha256:363e05fa78e15116c3c32c210ee36884fd6b9afa6d440e47112c3bd511d64cb6 \ + --hash=sha256:398aff33cee2767e3e781d2554c54bd0dff386bb437581e0d8011fde1a942ec1 \ + --hash=sha256:3d22a20b1fb1632cc72c22f95f7b0d2961c3e1c235f245ba4c606c4771035659 \ + --hash=sha256:42a494cee34437f05546455144f2b5d9ac09b1face62bcfce597d2e521066688 \ + --hash=sha256:42e2f76b9455f5a9a844f770bf3e200ed3da0e15f5df3db9c31fe80b04b3d004 \ + --hash=sha256:42f6930c31dc7f50732c9ae793c2786c7b6b044195967bbdde40bb9be81c4cc0 \ + --hash=sha256:456a61fa52d579ebf9df2e9552ead5129855dbaff6c1e5a9b1bc408809bdc062 \ + --hash=sha256:471cee653ae88de62096552e6d24ccb4a5adb8c8c9f10b5054d0122c15bf2779 \ + --hash=sha256:49cbc70e6542d4ccccb936558d1064a8012541e78f821f955cff24e357776c94 \ + --hash=sha256:4a7c934f7360e8cd64fe9efadcbd10c7c6364f531e432b9a4bf5ccbc9e0e8b50 \ + --hash=sha256:4be96343e422f2dfcd12ab5c9f5aebe03f82f737c6bffeca6830b3875cb44aab \ + --hash=sha256:4f42141fc14250de6dde5ee7ea4432be017252d91f19c5ad043c084cea629cac \ + --hash=sha256:507a24c282e0f42f8ed737cf048572cbf580468da5555764a8331735e9c736b6 \ + --hash=sha256:51b31d1c98274844cfd7838ce00bfc27c7423a4dc00fc0772fc3331c2cc90676 \ + --hash=sha256:58acb8ab8e295e6c5ea12f888cbb13cf21511ef2a3303a23f4325c29d17fe5c1 \ + --hash=sha256:5a59cc1c4442bc3d5c703bf720b51138d0bfc173618807c9ee2490a7541dd3d9 \ + --hash=sha256:5bb4e7ea95dcd6a014a6fef62e62467d67d8e582326443f3d68e71d6320a9fcf \ + --hash=sha256:5c58fe613dc5e5336357eff555824a314d8e43282600435c8d1cb6a7a2fedd13 \ + --hash=sha256:5e7cecbaadb83884793e05828cee59b210b24583b9c7425d0ba6a754fe22eb4e \ + --hash=sha256:616f097f2fe415bc92a247f02e11f634e1f9e9a83d327e3c915c15089c87869e \ + --hash=sha256:63bbfd5ded17c4840ac07cd8f1c21ba9d9708141f840b324f422f41b207e3973 \ + --hash=sha256:64faea20f4e2613363a1a9b9c7dd73058f3ecd00133a511e72ad7c511658f527 \ + --hash=sha256:661c298b4821edebead0c91edd2b00374d67ad7c5a1f7a91d4442633b79d6a72 \ + --hash=sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890 \ + --hash=sha256:6a8dddef476fab96d066d578fc88526767b836ab5ab21754e1d5bf3879c31c7c \ + --hash=sha256:6e192623c49c94421616a5778fba35cf0d5a8d000650c1967ef4448ee5cdd990 \ + --hash=sha256:7225e4514edb64eb6740324353e0da0711954fd8d7da4576755b1c6e09b697cd \ + --hash=sha256:75f80557d1389eddbd0de2681f6a390a0c5338c31ddaa821381c203fc3fd50d9 \ + --hash=sha256:770de9db11e84213beec501cfcaa013b019820ca881e03344dea5844f7876d94 \ + --hash=sha256:7750c6449dff7864bb9bb27ddfb0267756189201a3afc911d82b3caacd70dfc3 \ + --hash=sha256:7bde5e4cc5c10140859842b9d383af292b22639a4dffb725314baf45968cef80 \ + --hash=sha256:7ce713ace7c0e4520535b42b77eaa742c16dab813978064913e5a3cf82973b41 \ + --hash=sha256:7da0c5eff80f0197f3b3d1232ec5a682a9325f4ae9016a78f5f5ca35f9ced1f5 \ + --hash=sha256:7dbb61fe3a7699468030f71bbe5f8a0e326a151daa91beb11a6fc1f980c55e1c \ + --hash=sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a \ + --hash=sha256:8ef53b2de9bcb9197d31854256575d59dbac0cba72ac627bb291ef5eceb74be4 \ + --hash=sha256:937c0052c05a31ca1daf18de3158eed4dbfcb9cc107adbea227728d647be701e \ + --hash=sha256:9d2055050ea716bd38b7f7f1579c275386646b4894c155a3e2f3cd62ed41b7c6 \ + --hash=sha256:9f8d177621de5cb38ee3e731eda45d421db093ec0739f46a5594babda7987a98 \ + --hash=sha256:a2d7755bef5a12ed488f4ef1f1b69ee9191d7396083b755a5d2295f6edb4768b \ + --hash=sha256:a48d62ab9d6f4f98c983223a547af44be6ca3691074c31cecced6facd3ba2dc1 \ + --hash=sha256:a4f00aa42f75d6e4595e8866e748cc1705adc0cddfeb2ca86d0d03993d63ba03 \ + --hash=sha256:a6e721d4b0e45d5b65e87534470e67b18dcd092c83f68fba09f152b9cbc061af \ + --hash=sha256:a730a083190634c65cca36ba5f489531576ebd79bcd5c8e172130f6453127231 \ + --hash=sha256:a931079504ecc49efed7744c476a5c343a92fabf66dec2db95edb1b2fdc770e2 \ + --hash=sha256:aa9511c62d14da7aacc9b4bf51f3f697a621e83b2d6919008243c3aad168eea3 \ + --hash=sha256:ab36d55f9ed2d067327667c2fea18dda018eb628dd6347aa01dda6cf1f5d3836 \ + --hash=sha256:ad2c86c495b899d862ea0f4b42891b8713a3bd45dd4105c7fd51c2a72f39f3a5 \ + --hash=sha256:aeae0e330c9f6acd681f647d46cefd30c29f93e3392882e792e82080c9691399 \ + --hash=sha256:b0431303acaea1089ad4b3e9ce4e6518193def1118d4073ca848635ee4ea2e96 \ + --hash=sha256:b5bdfd1c873d4e093aabc0ca84c4ca6dbc4f752afb5c86f146d9742580c9da2e \ + --hash=sha256:baed1e86cc735622097354b9d1281406caf42ff42a886d29faa8e8d1630333be \ + --hash=sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf \ + --hash=sha256:c26608d2222fb1e94487e4a387d85f13eb55d5ed725cb25a0c589ac4ee60e7bc \ + --hash=sha256:c7659f22557c5a0bc4855cd635f55edec690cc008a40768527762cb9fb263455 \ + --hash=sha256:c8c69575568085ba0b1b10c0249d779a214aea6f6522e949a0fc9fb0fcb449d0 \ + --hash=sha256:c8d2c9fd1f2d16f780d15127abb050d13d1a76c03a4bd87d7e4980e45e511e12 \ + --hash=sha256:ca82be1a1d406ecfe1d25dc16cb33488e5a16bf4438c9fb590484ea29d92478b \ + --hash=sha256:cc572dace3f60ef98d7b12ff411d20f5362feb31a0439eab0085bbfd349982d7 \ + --hash=sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692 \ + --hash=sha256:d28630f5854ab07ab1fd4aba756de52326c82e6be15d414b12793f1975048b54 \ + --hash=sha256:d9c275eaacd24aa73f94ffd6de08fc3f932424d8b6c376f4bed7cde376fe7bc3 \ + --hash=sha256:da0e573f9f97159390c89d9f1a9e41908b66d408cc5b58d08cf3847d844c531b \ + --hash=sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be \ + --hash=sha256:dddad92b554513a31f272570678ba307fb9f618f05e3d4a5eacafff9eae03e1d \ + --hash=sha256:df423d40ee8654634421812bc3b196da3f9bd7d32929da813f8394c4348a5358 \ + --hash=sha256:df913725b79db7bcf03448f36b7bf8815363417d5b58deecf9305e3e30f0f21a \ + --hash=sha256:e0bcb7e0f677f543555d2adff3bf19c05f66cdb4796e5ff602442ab2fe3c4ef7 \ + --hash=sha256:e2d65b31f36619cda3999b78b2aa9632e76b78448e7a56fc4240824200e7c4fc \ + --hash=sha256:e6e8cff14d6fb0be70a09c0bdc58096f501952d04624ebf867e0e56da2df8960 \ + --hash=sha256:f16c709686a78c727bbbf059f92b0bf41c6fc60deec706d2dc19f529175a6125 \ + --hash=sha256:f24fb43132a4c6b4cb4eb029492919b2db645be6808d738f244fd146c03c32cb \ + --hash=sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a \ + --hash=sha256:f5cfbc5fe74540d335175b656c725d74d90e3730c626d92575eea35029d9afaa \ + --hash=sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf \ + --hash=sha256:f8ec5e643a9a937f64e1999eb9f75d072263751912dc5cd06d3c85f8f44be7c3 \ + --hash=sha256:fb92203a88b3d3053034db775110081c49d28be6551923805e039924093761e4 \ + --hash=sha256:fcd22650c908d7b7da162bbfaab594a1227a15d1643a98c68b122ac642fa2264 + # via cryptography +charset-normalizer==3.5.2 \ + --hash=sha256:01077390b03f7988f11d700a2194e69b119741a86b1a638b1db88891e3eced8e \ + --hash=sha256:01b0c0d2262a9e28e8484a278c7e1b5d650e3ac8cf2683d2967e25899f208bdf \ + --hash=sha256:04851f73ae72b8413dddadb16a49dfee95263553741fd42d546f7d66907e6be5 \ + --hash=sha256:0521c5665880b33d603717defa76c094048900010897909952397feb3039da56 \ + --hash=sha256:0774bf9bf620249fee3e0b8b9fd3065de213be30f3aa94ce2494b3b638949e26 \ + --hash=sha256:0891b9d3903c5571c03771ca669a4b0ec5618ca722a5c957d3d29cd4e5062848 \ + --hash=sha256:0c951d5e6dd9c2ff60609476752bee49da4206adde960ebc247766937f72e718 \ + --hash=sha256:0fed1d06615f022ee3b13caf5e8b180cfea32bb2c5aded8a9d44277afc040f93 \ + --hash=sha256:114e4d0c92d618409ed82a99e22b5c5e768fe995f2973f78265f4524f49d4640 \ + --hash=sha256:11912e4bb14baae7c5d8791aa55ba0a3a03ec6729073307b0f57270abaa713d3 \ + --hash=sha256:11a4d68a6ecda3292cb1e50239e111543ba5d709bb62a6b4ea1afcfa729d8875 \ + --hash=sha256:124fbf1a8ff966d87ae05bb8bd45a71f966055ed8bba320d0c7cf450bc5f4d0e \ + --hash=sha256:1461ac396c4fdb983a675f20aa555624f0ee18ac83d832b9244ffff3d8055275 \ + --hash=sha256:1503bccbeb36d5527790c3930327704c39af22de3112f1b1666a9f3ce15ee204 \ + --hash=sha256:15bb4005af6320d259dc7593ca84a38d7fe06a421dbcf7b910ae23979101e787 \ + --hash=sha256:15c44f7edfd477b06f517a5cc317fc1707edb9de2c865f43d4b6513907473234 \ + --hash=sha256:16fa0eccf81304b79c5cd87f9271c3b85dd9dd99245e4422ae9c0dd45e0f99d3 \ + --hash=sha256:183b88127acdb4fabe59d951ab424faf1af7b63cdbb5f776186c1ea2ffcaed98 \ + --hash=sha256:195c26fb65950f8fce54e26349852b7bdd7c5f120aeefbcc440b8a20faaed4a3 \ + --hash=sha256:1afb975bd5d68d5ce9f6b6d44fdf2f7e34b895a35e95708a7a91b20a3b51d187 \ + --hash=sha256:1b4cbc7c3491ccb4aa17fcd8165649d01cf39f76de1696da8631b5f71b85401d \ + --hash=sha256:1bc0baf5ef96b6ede57d47f4b8fe4d9d84019c3bfcbeb20a41edc6a6ee341f1f \ + --hash=sha256:1c50fe28bbc2ced33386f298650d91218076c05420e6cbd790b913adc41659e7 \ + --hash=sha256:1db38f4c5496827c1a501846d64d14c3b80c7e6714e406cd7dc36a9899fa1011 \ + --hash=sha256:211d5a3eb6af8f513b8d4ca19a8c1b7accab1b5f0d3175f9826b03c1a920dc1f \ + --hash=sha256:23851fb4e1b85ed3f6c2a27b777cdfe2e19fb5b38429a8faf38c7542b7665869 \ + --hash=sha256:254eb48b9fa5ee9898a3c445825a1f340fe53712a098904b39b0bddba8ea3cb1 \ + --hash=sha256:2625388c6c754520c37abaf3b41eb34d1cc4a373f457898f08606c8e362b891d \ + --hash=sha256:281cb91036248400f4cc957495cccd44c275c2e0c5854f7e45ac5cf7dc193847 \ + --hash=sha256:28a15fdad492a99b6eccfaaed66ef3f74050680545ea61ec8b2f4c538f1f1320 \ + --hash=sha256:28b4f0d66fb834ff90f28209ac7bce77868c45d8c93e26f906709d9b7c2e1af9 \ + --hash=sha256:2a925889534b3748302dae5dead07cc13480de1dac3aea80a941b729b471ef93 \ + --hash=sha256:2b7b3bbfb4fe8ef40600792d762fbaa9057559f9d3fad209525b7a22b99e91fd \ + --hash=sha256:2c9ad19a6cfcd5ea5c0d41161d22f9df1dcc277e9bef2751391334546a314c00 \ + --hash=sha256:2cc961b171b3f3440f410489ab3573e86aea8736134ebbb40ea1338b7f0831bc \ + --hash=sha256:2ce45c6627b22c47e390bc91a41c3d13032192e699fa0bea96e9671b373d69b0 \ + --hash=sha256:2e06a3a98f916dd41d27f3105e02e7a40181c98c94b9158733d03a6f80506c09 \ + --hash=sha256:304d5463e65a35d7bb0850550e0780395395f6fcf452f04db7d5ca7cecc425ac \ + --hash=sha256:304d8e4d493af723536393eee0c689eb7813f4a474c8b479dee63f1fdd98f621 \ + --hash=sha256:30fcd120b732aa79317f08dee04d7de0847822e4cf7ee0e9f445bb958832252c \ + --hash=sha256:31f3930700408d211f13378ccbe1c40845d8da54bd0681fac3a9b5aae81c7aa8 \ + --hash=sha256:34276fd796040bf0993ab33a369aa572e6979c7aab225a88893667ad8eac8f7a \ + --hash=sha256:355ad8011081dec5412240c087a9a0c9d4d5039f3ed11a3f13e18c2b29b56c51 \ + --hash=sha256:38a873987f3be698494da8b2e3085e29da02da7b633dce73e79c699a113d7bf0 \ + --hash=sha256:39de2a259fc954455c57274dc94c79d5842774e1247a016aff30bc0efed0f4ef \ + --hash=sha256:3d14b50de6bf4d0edf857a9386836846f982b8f524e188e2e68b96d702bcf4aa \ + --hash=sha256:3d21b8b13c7592db2ac5e544a6d83187b995257472b0c9e8351b6d507ae37ed6 \ + --hash=sha256:3d31298449090ab8d47b7b1b2a555ff73cac7ed438a08b7ac160980c7ebed649 \ + --hash=sha256:3ddacd27458c45bdacd6bd6db644bfb730efbf9e830310186e3045c9c5be8fb2 \ + --hash=sha256:3df041de8887954562c9b261cba85ca0e9ded74048daf125f45edcfaa4832229 \ + --hash=sha256:40ab6bffa02ae10a0581e6c198be7d2d8ca5c2a0c64e4ed3465d766df457573e \ + --hash=sha256:4275811936e2f06feff5e598fb42a1b7ae852da8e39605211892b56b81a34efd \ + --hash=sha256:443eae2bf318abeaf6f15d785138f71fd6de770e99a92158b8b814265e079115 \ + --hash=sha256:447441e76ec720b15e64418d32e092297340387053047c7c694f579efb0ee1d9 \ + --hash=sha256:4495c5002a7b28557e7e222e77e0b661183e432b7d6d2e788101e3f240e05b8c \ + --hash=sha256:44bd4fbb29dfbeba60e7d2bd000c59e4b21ddb3cc53912b14048d37092706d7c \ + --hash=sha256:4685902cf26edf013ed7a3da0f426ebba7a00ebb9541386d835afbf002c11cab \ + --hash=sha256:498dc3188ca05a68231ac3fdbfc7f57eb67e1343c30e0fea17f8218c1599b253 \ + --hash=sha256:4c2b5031f63e331e3839b40aed2dd6f191e9c07edbde303e7876846ea1946995 \ + --hash=sha256:4d48f2d08b9de5864e2c8744d4461b862fb149a18274abc8b698c45975573438 \ + --hash=sha256:4f87960d57feabfb618e4e0af6e7371645fa26a277860739d6e5d6e0012c92f0 \ + --hash=sha256:50e3adfb96fc189eb27b1cf62d3b598b89b4bb0420d93a3d3e42e137409011be \ + --hash=sha256:51cf45226a9b588d0d2b4880c62d686934b63ab0bd79ca23ab0e9762eb27441b \ + --hash=sha256:52aa6992700996af31f375de0c6bacd402b0097fe40b53c426b9f51a90ebabc7 \ + --hash=sha256:55ea99acb17b9325618de155a0cd6a2e8f5d10be008113e1d433bbb58db543b2 \ + --hash=sha256:56bc200a365efb37383b7852e4cc5898d3b2da5987289b543956cf8cad71018a \ + --hash=sha256:588461c2e8384d309bd63e5826019b6977bc66d629b99ac8737bb795d7b2cb5a \ + --hash=sha256:58ca3755ee7ff7f59b57789ec9833c9de9ea275405cdd240eda1f193112e398a \ + --hash=sha256:58f361dcbab699cf8f42db3f47c8e7fd1036f138c23a5d08de9fde5f425a730c \ + --hash=sha256:598a11a2c7ebaa5334bf698bf29568c9c390abac6a154d8170fedecd1cea38c5 \ + --hash=sha256:59f63901b0031c3136cf64704dcb21de0bbae62ce2c9529bc39d27665463de37 \ + --hash=sha256:5cde776b7cc66e4f6c99612cea4aa7269aa65863f7a15841b2c264f103822f4e \ + --hash=sha256:5e2b6b57e9733d39f0c9fd3185efa6b8e29652c4cd8fe94180272cf6ed9a78c4 \ + --hash=sha256:5fb29fb8cd1a46c27a1bf9613ad5ec2599310d46b4025d9556404a6b6a292800 \ + --hash=sha256:6045373d5a89a5ec71afde535db987ca28e76dfa276c2d4c818265b375d4b055 \ + --hash=sha256:619799369eeef6366ed3e8755a5670f4f2f0fb6b30a0fd7264dc0fdc2357058e \ + --hash=sha256:62588a277bfb59def052abd940703fa35107152bf479781a878617d60faf8fb5 \ + --hash=sha256:62603db9a7caa0802eaa28c1c46fecd7b3a263a774069c24c3c28c302448721c \ + --hash=sha256:65cd72beeeca9d3aaea1201e5923859f308f952f9c71de93f06063c79f0f7a3b \ + --hash=sha256:68eb192d85ab8e5f6ec69c2bc6ac0179fbf04a5ac1569d12fbef74883fe102d0 \ + --hash=sha256:6bd128f206a7752ae1f2ab6c61bf8a24ba28913a10df8b14c2637b973ff97a80 \ + --hash=sha256:6be488a102b8cf28d0391d8c4ba7748938ae28b78ad901f8585520fca33ead1a \ + --hash=sha256:7218e8f32b0956cfcd048fd42d9d5779809745ca1d86113ca56f66e7ae1549c4 \ + --hash=sha256:7441d755b7ab94f8d4eb3e43ec05482d760842fd263d003a99102d742cd835e2 \ + --hash=sha256:749e97e1b32313717a565abbe321bc2190bc8b35f1a67e4cdbc7c56c8d8ffe58 \ + --hash=sha256:75a3ceed0724d625d64b86ca20aba182e4df462e04c2414fc941c0f523f06aac \ + --hash=sha256:780fbe7cab297b81dad9fb8dc5eb003c0468ffb0d9e5f65068c53a34661a96bc \ + --hash=sha256:78456a747de8dc58360ffa581f30a002baf5aa28cb262536545e91f113ed7639 \ + --hash=sha256:7967d08cf06dee78443b874f98c98036f624f3a4e73e11f9f64f5be4d25393cf \ + --hash=sha256:7a881931aa470808df94a8c380eed2bbbc76cd9dc622310f99665658c821eb6d \ + --hash=sha256:7dcd882da75ef9adf94903b1e3b9419e8aa8fb4c7396822b834b9ef7fb96954f \ + --hash=sha256:7e841fb9010836c992c9f12fcbd43a831de93a5f726fc1ccd8ca1d0268c5014c \ + --hash=sha256:7fdde2c9fd9e3eca40631e024664cf2584272cc8f96308cbe5fdfc930f51d8bc \ + --hash=sha256:8024d00c3faf3fc0c16e07a69f4405e8eac7cc0ab15f65fe6cf43827c4cf72b4 \ + --hash=sha256:80d02b6f04e92601a081dd97b23d3128033098bff5d35d392ddcc0476ea11253 \ + --hash=sha256:838dcc90063569a0448120554591a1d6c4a4ffe11babf048908793154ab86ade \ + --hash=sha256:849df64e889b2e17230d58410a03dba311a65b163508fd33679b2b737d4b7858 \ + --hash=sha256:87475fabc8d9996fd9c27debb395e642e8c838d78a00b6e932227a0e06b81e26 \ + --hash=sha256:87e50a3e7cb90af586b6c5faf23e302a970415ac73bd7bd90a515a04b427ef96 \ + --hash=sha256:89b53f3cda69831909888e0494f4fa0bcd3537e3e138dabeb620bd6ad946bae8 \ + --hash=sha256:8a893cc101149f80a653f82062ebc95b34525a2614382e1da5458fe7c6997249 \ + --hash=sha256:8b2bfab86aa71ae13aa41a6a26aab338e0db2b8bc75434b05aea89e011ff35a4 \ + --hash=sha256:8d86d6fc60743dc916eb79e2eb1ec4818e21e427731543af40a3021851174a13 \ + --hash=sha256:915563965d418f986e7e145accc592eae9e1a1be3566ff98a05d7a9ec42a76e1 \ + --hash=sha256:92888bb3187c5ba50500b00b3b310c9f2c651709d28036077680cb5255450a03 \ + --hash=sha256:93223adc95033dd47133a46ccfc316a0139176fd79085762e27202ec56018f03 \ + --hash=sha256:9373ad13ef0d2c0fb761e04e55bfdee5a08b52cef2c882c8fbe9935b1517152e \ + --hash=sha256:9409a8bf35cf78353942504b24a57de3d75b708997a1e4bd8db71ac8633ce364 \ + --hash=sha256:9b7f416ff0978e2f2249330527f0ad6fa02f4932e6199692d3b52da2048c19e4 \ + --hash=sha256:9bde855991b7e362c146535e3136a50bfaffc0487d38b33ca7e5edefc6e23849 \ + --hash=sha256:9cae88599c7219005d879f98e5ed53341e9a122af585e1091200358a3003d2a0 \ + --hash=sha256:9cf9b1a857e25c4baceeb3624e92a56df3668f398c4acba74e174d81fb4d1d3a \ + --hash=sha256:9f56f72050826f63dcee7a7f55b0a77168cb3bfc553fd405e7f8f9ece75a4036 \ + --hash=sha256:a090bb2c68df85450502e3e20d665e3a5af9c65a84d6508ed477badd49166fd3 \ + --hash=sha256:a192e2c40070d92c3ccf777e3a5c4ff515573cd2bb7ed0c537fdadbbec5bbf21 \ + --hash=sha256:a19a731138fc27d5682277d3b9df22855cea1239bce7fcec5f78f42ef2d1f3c3 \ + --hash=sha256:a66c3bc5ab1f0ff2164fc9965ddd611ff0802173f4b9d24554c563f6ab7e1d6e \ + --hash=sha256:a815775b6c38d4e0ff7bcffbeba67feded90202bb6a226b8dd35f1c855217413 \ + --hash=sha256:a89012d6d5476ee112d20d998570ed58df2260a852afb1758809cd6900411d21 \ + --hash=sha256:ae4f5fea5b8b8ccff88238cc8569303e5ee95efae67fa62922a311397a71f346 \ + --hash=sha256:b6856554c4f44d79fc2307d5768854310a8f0096e501c75637542c82292b0429 \ + --hash=sha256:b6b751274acb69d77b3323d6b7dbaa3c7fdfc1eb829b7eb61d262f32e1af9685 \ + --hash=sha256:b736353c0a625bbd5fcec108576e2385db3496f4f771f785ff32e108d3c3bc45 \ + --hash=sha256:b7fd005a73d9e657273b7a10dc71a9e03c8fb9ee6999798d6918ce095b81ac7f \ + --hash=sha256:b91363207bd9dc966a691e959bb47f64b30f7ac4b072be9968b366982f7db77c \ + --hash=sha256:ba0b1d2620edf869789c3879223f52bf2afc5d31b3cb47cc57b3a12c05e2aa9d \ + --hash=sha256:bbbfc8e28816f19d7c0f1816664980c0a9875d01b27cdf8eedddb639d9e108ad \ + --hash=sha256:bd16aabe4a02a297c23417aa17ac6299dbd8c49f673bcd645b4929b11f5a4400 \ + --hash=sha256:c0afc6800ba57ccc350374c5bd6150419915d95ce93cdbab2d783d75eaf30ecb \ + --hash=sha256:c6708715abcf3c73b99508253e961a9967f02fe536532834149574eda6de0d1c \ + --hash=sha256:c7c9ab723cde841fefb34efbad91e87f00a674b1fe1cd0784fde742bf2c154dc \ + --hash=sha256:c8f3d67aeaf55f017982b73683f0e7342ba2f6635a78f69ce89ebb26aa411e5c \ + --hash=sha256:c9790464842f85f437dbbb54417eda1e0e6bfc52dd8d22d6fd1c994b73b2dc74 \ + --hash=sha256:ca403d7e4798f525fdfc78e258820419cbbd0f0ecbab9de7840e3c017cf6b8cf \ + --hash=sha256:d008d90a7f2471519aef0c90dfbe73b3e6e4d5e66ac48e19154c17e89e98b604 \ + --hash=sha256:d19fbd981a488e22cd04883659ca6b08f50b5974f9fd7c95655ef6a043e5893f \ + --hash=sha256:d1befeed746d247c81127bb14de9dc3d30edb6e5976d34f83f86ed262b1d9105 \ + --hash=sha256:d2374b62878abb00cd8309b32af6c0b715cd02dec0ca74ef12e5069bdc64144a \ + --hash=sha256:d376bbd28b3a8999db1a103b3b388aee6f1ddeb3e51bc2172993efdcd86e064d \ + --hash=sha256:d4a7319f304a774bed22115bc891618e45f85065ab44ea6acd07d274e750519a \ + --hash=sha256:d6734d2ef8a50fbf8445c139477da401f50d62a0606bf00e20ec6d87773fefb1 \ + --hash=sha256:d760fe2a4d7c3b226cb9026d6a842868d52a7901bd98420e1baf14e80da85cf5 \ + --hash=sha256:d913de495d90407cd859d263bee2e5d1a4ed3eb6573c04e70d9ec619a7cbed7f \ + --hash=sha256:db19d07e2e0129e974a0e65d0064fc222a446cd5122c2fd4184d2af9fc734a9e \ + --hash=sha256:dca9ab98072a5a54ebacebdc45f53e645336b320c667410b061be1ca588ae709 \ + --hash=sha256:ddc7dacc8ece3a182e7f15cb862d1fd616b46d076cb1ae9dd232b2c38b655874 \ + --hash=sha256:ddf19c062bea7a0cc80f519243d2c01dd091be0cf952a0750d4ad576709559f5 \ + --hash=sha256:def79fa35ef0cef8d2accec024f4fdc7ead3012ff02f5215c783f39f03ef8cfc \ + --hash=sha256:df29a0a7107f7011e77f4eebdddec4c7331e24d787a0b21a46d63bdf7445da95 \ + --hash=sha256:e09a3942ecbdee5cce73ea9d42da82b81b72ac1bf031ce069b93b5adf4eac8cd \ + --hash=sha256:e242bb1c5e76e97dfa9e7f209a71e93a01d7f19ffdd5cfbb2e2d55b4f08f8ab0 \ + --hash=sha256:e243bd13217235fc7290c621941c3f5cc8b66e4872495be821d7436ba2fb838d \ + --hash=sha256:e2af3aad578aa6bd1384bcf4750fc285e5a9de53f40b7d41e5a0bf748edeb2b3 \ + --hash=sha256:e4e81e09c1578b8df602e3db08b0b3ea0a6947ad612f52bf8dc5ea8d47691f0c \ + --hash=sha256:e54da4baf05720032d527874d40b65fa4d7e5c6c6a43d0c3adbeffcaf275a2b3 \ + --hash=sha256:e80e6c2f55656b4824d72065abb4ddd6a525c74bd78a0aab5d9fc2cf4fb5af50 \ + --hash=sha256:ed2a239c0ea213acc1908150a3037257083c7c083128f1a4cec2ec4b97dca491 \ + --hash=sha256:ed905975ab14056a2e5eb1c376cb2e1ebc5396baf84163939c518556fccde9f5 \ + --hash=sha256:ee21e28f0430bd6dc9086c6e525d5e818a44a5ad19720c8a0ef766792f3eb5e5 \ + --hash=sha256:ee43c17b173d46a3212baa6ead3ae258eeabdae48c263a01ccf0218c366dd655 \ + --hash=sha256:ef4fcbf3327382cd4c9f540babd61248208af7b93eec4de397b4d5f58a09e288 \ + --hash=sha256:eff0ac9dbe711a4aee69bf04a83896aa9b85f19641264053a9f6d48573abb7dd \ + --hash=sha256:f0aa869112ef88429ae17820d99c3dd9504c9e9c671d3c246f3d7442cb051084 \ + --hash=sha256:f3c96f633825733f735c5a9cf21d21a257d8e1edf0b1cee0a064b9c424ca0f7d \ + --hash=sha256:f5833ad231be5eb6553de524a70f48d71b2c8563101750531e0b80184e175cd4 \ + --hash=sha256:f5ec61164adcec446f8969a3358ec3f9b26bbda3b9213e5586d219afa8df2915 \ + --hash=sha256:f7d486c83842422badd511868fd8a9a20e9407ace71564b6af47ce7e60a336c1 \ + --hash=sha256:fb9e68df06293761f9fe66ade60a9bc6d0f5e42b8acf2939a9158af86ab0e5bd \ + --hash=sha256:fc14a032f813bf5fe624d991960ea83e9715adc27e4c1830a2361eb1d02ac341 \ + --hash=sha256:fcff63213e8e6e47770541a4607175404f47cbb3ebea7b6058cc82d524a0e424 \ + --hash=sha256:fd1fbe0f116b6e55da77aca2c6ddcddcfac2186cbf78bdebf40fc156efca389d \ + --hash=sha256:fe9753dfee015c570d73df76f899f18444d41388bffcde097deba51c4fadbb9f + # via requests +click==8.5.0 \ + --hash=sha256:255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360 \ + --hash=sha256:ba0d2089de75ea0310e2dde03160e6ca10009947fb95a182f9b54021bb272e34 + # via + # black + # docstrfmt + # spdx-tools + # towncrier +colorama==0.4.6 \ + --hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 \ + --hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6 + # via + # sphinx + # tox +coverage==7.16.2 \ + --hash=sha256:00d3eb96e9988c45f50cccd1f1496571ac5c1f91386ac02c4d55516eeda19a24 \ + --hash=sha256:01c6908bc613b420c26c818fe948e1b97dfd041a53c98b01c63bd8321f5c9aae \ + --hash=sha256:066429634299e14dd2d511e1e85f8f9cecc500781f6b41907c0dd6f1baea7e63 \ + --hash=sha256:0993d0e90858c03943d3cb152e068a20dd4707924deec84dd2230261baae3b1b \ + --hash=sha256:0dcbcfcc059117284c603ff8cb61a65872512882f84a8cf0339241f7f7c2f148 \ + --hash=sha256:0fd7a86fdda7cb6d616d178654bd0ad6bc0f3f33c2e478aa598500a1a9e34eda \ + --hash=sha256:11d28e9123a9156cb405d8d27b44256c9a58fb5decc2073a8f17862057e3aa0f \ + --hash=sha256:11e597173af1dc33d5f8a7332ada544199269a223af1ee1770ddd5e245ad0fe8 \ + --hash=sha256:126d1af8804d7224421fe991ff65d3ce649081560df7a98b1a5ffff07f9923bd \ + --hash=sha256:14253fc7bb15749b849795a06f5d3b6d8bc3fb8a4b5ddc341faf7a89dce205fc \ + --hash=sha256:152877cdc8a07264882cfcd503ba56a3ef6cba56a70e8c70f6eb8ffd7384789a \ + --hash=sha256:17228fbca0f22976f797be94e975dcd237799c657d49551c7de1e0654d1202e9 \ + --hash=sha256:191803c4996b499fcd78c2ad5e5f767dcc53cb4dc6de6d6a741b443a1821ef02 \ + --hash=sha256:1a37c6e478cf687e1aa30a593d19c92c02fad9d122b51ab73f51b8dc7a0c0fc9 \ + --hash=sha256:1c569a9fd25505f1cd6bea90588818f90373ce90e2632e2cacf19ddbd6e14fdb \ + --hash=sha256:1d56e4d21c56d2046447733f8b118409597db48c01efe898ee9ac24e858ec2d6 \ + --hash=sha256:1d5d0e3b660506fb84f995814e3118a21efdc0c8eb80127da1be627d90093c17 \ + --hash=sha256:1f15254427c9b33eedac4f198eaf9e356eb4f6214551afb43da6194a2c088ad7 \ + --hash=sha256:218d742afca2b5ad5ca759e93eddedfbcc6eadf8322f080dcefc40b7bd4e2d48 \ + --hash=sha256:22957cef43ce038641de78ba995de7568d2d6a37c6ddbf7fa0fd7d1ae2344d91 \ + --hash=sha256:23219888477edd736b6fcaec1272d47d93b926e999641ffea7e53a1738e70b2b \ + --hash=sha256:251aed777c47c77aba047096d4542889db089227655711dfc2b9c54ef0e15e35 \ + --hash=sha256:28ff850182a67d117990fa2ce5ea1032836d8c9630dae867e8bdd3bff4533b79 \ + --hash=sha256:29309ccc86b7f33df7db12813c299f215bbbc470ed6292d0bedd63ffae1ebf64 \ + --hash=sha256:2aca0bdfa9e91621d5b09d815357bf63def4fc0e9cb66da67bf2cf93f3b1a6f5 \ + --hash=sha256:30c1b65d529e46569899fadca59e4a87c1faf2886923f1307ba61e654d4f3c20 \ + --hash=sha256:35f37886699cb9abd29958247d718628d5bc6f39e623dff66a09e546c42a7e03 \ + --hash=sha256:382d3346d56b0eec1b793d53a4c88799c8053f516aa3a8d7c44315696954bacf \ + --hash=sha256:396bb16e04ce04efbb3df91456ae4e3da918e69ecdf67fb711b0a0fdf35ccce0 \ + --hash=sha256:3e7f99698ba3a7d13988bdd984b7ebf13af4dbe2166dc8502eef90d77603b0a4 \ + --hash=sha256:3e861f1071dcc2fec1e88bef0920f6b1eaa66a143555b4f8ab79ba2b0f30ef55 \ + --hash=sha256:3f43bac1856ba269b905302778d4df433d6006489a192174ad77ac528e395032 \ + --hash=sha256:40c0f00899fe6181ae7f434ceb200e51f5ee4b8ed10e3b5f0b605f0cae15da87 \ + --hash=sha256:414c26dfdb96aac2d570a54e03008f001e32eb2d413705365503648c6bd361d8 \ + --hash=sha256:4358b9c8c0125b460407f3017c6cce8156e904b32772c5630d27112f52bdbfe5 \ + --hash=sha256:444889f7f66b74e4455c0a97e0e166dd41177f1dca8c0239a47cff25e05ba7e1 \ + --hash=sha256:44f21e407b278efdfc1ee5e481e00518bd1d500310a30a5fbf2bcbedfef4aaf0 \ + --hash=sha256:4cc4f73aa3fabc36e32046d6cd2971405948d8a903636508a3d3b2f9128b3a95 \ + --hash=sha256:4dbbd1155ca46e6e0b6b89d204428c56ef6a459af21333f365d135a2820e5a09 \ + --hash=sha256:4ee546b9e4872ffa194bf07ac87bfa1202ebb824d0795dc1ef22f175545ca90a \ + --hash=sha256:5139009b5efd2194fc168ee9362f0e191ba612ef5d29242f9269c22f9b8f80c7 \ + --hash=sha256:5375ebd99038021b35e99dc88255022912c06565d316212f4a576e4b08d30f5d \ + --hash=sha256:5397e21a90dde0e9c6896b77ded8f0be26b66f8b22b33aed41f6043ed95d55e6 \ + --hash=sha256:57ff3783f99d75a1e81dd56a9737eb5665e6736a5d93258ba596b6dcad8fd05b \ + --hash=sha256:58d4a54c6ea672afef66d49be922a2c69826c5ae1a42a9cd94f0c9c2bacdf800 \ + --hash=sha256:59c3926585e1cd1f2190f4b2ac9014de1bbeaf0d5d0587b0dc6b0aa90d17896a \ + --hash=sha256:5a27b731c171e43dc8b5f32b76a5051dde2ec9b9366c87028f08a7088ebc2c7b \ + --hash=sha256:5b3146d2317c75f70df2509066d979dadd941f7021cdf9b5db4bcd8568258e25 \ + --hash=sha256:5dca0bb66b4c3d624ba047887bf70270030c150692d543cb501293dc38a9f4b5 \ + --hash=sha256:611a44e5229a59d7483ce830160e1a0e85f700562c7a5651c7c63fb8f4eb528c \ + --hash=sha256:648352b94507179d82637292e7ae8802508d95f78e2f00a705a50b6c48011681 \ + --hash=sha256:6a75180829efb8ae62b4aded25be6ddca1c888d138d2d82e21d93bfbd88f41cb \ + --hash=sha256:705e5af11d34647efdc170c7840b6857c81cf74be96419a553f237e68e62cb72 \ + --hash=sha256:723dcdab91357159b722935b500ee8abc0a66c8c432e1e9fabf4cc7598952de8 \ + --hash=sha256:724bd0f1e81856b35e59fc98cf7b4e544a3cb662e4e0864dca73d4326ee9d808 \ + --hash=sha256:732d950e51f3ba4fb6209c73250f3e8924fefca42953ee04a9e65d8c02414d7d \ + --hash=sha256:736fde09ea39646d11f8e3b76bd3425c075aa4dd45f24891970bb77c14ff20f5 \ + --hash=sha256:7a076277ca9f5750cc230f0f578ebd2620cec60255b25707361699fef6fb465c \ + --hash=sha256:7b3bce4a0d05401d70b7d0d5ca783e686bc9d30e81dbd7d980d532609bf809e4 \ + --hash=sha256:7b451c68218c150f616bc9649783ec8de76a59792c759b43aa0c9c0466a465e4 \ + --hash=sha256:7d0732c83746bc24123c581a85d9dd96b70ddb538c9076020aa1a041790361e9 \ + --hash=sha256:7ed238d227e23cc300c3d464babdaf9f6ddc740aa1b15a77ae96136e6a7c4516 \ + --hash=sha256:80d3f7b48d43ee8fc5e8707a8adb43d743a5a1a85256c25a24f9d6d0e2238fa6 \ + --hash=sha256:80e9fdb4c3d926b6ba721d4bf7435bdb869c3527ae7803290361d0ab73db13b6 \ + --hash=sha256:848893e1d361448c113dc2f0913503522a6f7be231d0e38333d2a22d9698a011 \ + --hash=sha256:893ea9cf86cb8d2546812ac93d973aaf2ee1fb45110a873b014214fd23e3725e \ + --hash=sha256:8afd9bf35cc6a1f22eb3634808fa8e0b91902459c5721ef2e4461dfe771d7f08 \ + --hash=sha256:8be099e979fc42559328a21828281b4578304191ae46ed4e80a407048a82eee6 \ + --hash=sha256:8e209591f7c41ae4a9171335cf6156afda0b21de73b02f73f5aa95b2d5fbb08d \ + --hash=sha256:8fc15cc8d0d06e873c00ef18e1372d605f9aaf3de27d8c24e50782e75bc8b843 \ + --hash=sha256:9174f0af24e5eff248b9dbfe76ec5275a3d19d37edbc2810543f12cf97347a34 \ + --hash=sha256:921415102a90637fcc2e3f169f61dad7699ecf690e8639fc21b813acbedc0967 \ + --hash=sha256:967d72c835d7a8cf0af99ec813a2d06e3db6df706402f1fe85b31b437645f495 \ + --hash=sha256:98d9c97f51b334b0adce7b964442a9af33c1a00c6ac856984cc5dc8d18f81c75 \ + --hash=sha256:99704f73721e23859112072d522076e11c31744fc96b5652e5dd2018aa4359f7 \ + --hash=sha256:9a75a4704ff640e46170042eec1f984385a121227c505d5a16ad8e495f452541 \ + --hash=sha256:9acc7f7ec4a1b5f89bd929fde5b8a714f6fafdc6cc18725413d510aa082b47ad \ + --hash=sha256:9c6afdd69218202bc1758c9a14b86b8cf1084f37ed2ca143e567a103772b16d1 \ + --hash=sha256:9cdf19874e0d247f32f03609200370343c3c7aa260b191d8c2bb251d36198283 \ + --hash=sha256:9e1d0ced76318bab499693ff25f64faa343415187cb2e4d7befdfdd391a1cf6a \ + --hash=sha256:9fd670ac43b709c575aefc25bf52d8a598a3bc5017bddfd0a179152ab06a2deb \ + --hash=sha256:a0f2285329dac10ab08f79cb11f5692c497018e6c7c511f95e6fd63a70b8f831 \ + --hash=sha256:a2fac6895eb299a2e52d7bbb8fb3903502b9da8d3f5309ceb16ec40c646b58ee \ + --hash=sha256:a336eec40e3520d369b8a6cdabb4f596e69a8b42927ca074aa1452fed943238a \ + --hash=sha256:a4624f80732f6b427ac58f1f59c577a0994a12e8174b5af6a027b4b58795d4c3 \ + --hash=sha256:a56ac4fa5a75c7e182e8f62600cfb4aff43c5ed7356a034f3557659c3bec1d90 \ + --hash=sha256:a678c0b6b22086ec2427359d22e37445d4a792f5fdbbc744112c7dade65cad02 \ + --hash=sha256:a740ea6f083c6db7b926534d159508f80ba275ab35e722522de0d18d0f56e55f \ + --hash=sha256:a90700f743e29aa3d75a6ff5f01953176a889c00e526194bc4d281731b88d99d \ + --hash=sha256:a9a638be322a8d76a41cdb17781c7f82aaee6a66493d8ffb7e2c09ee22423d99 \ + --hash=sha256:a9cd3de0a5bfe7b0e21ee10e1a14e3d61bf52efc88217ab1d95d6ace6970bd46 \ + --hash=sha256:aa62c85046473959c13ba9edca9dc90a77d5c1095b1ba313556314d77fe5b036 \ + --hash=sha256:aba5c63b7afdc749cc9eae943d5b868cba2b261a176378fa1c5a30bc8bc89982 \ + --hash=sha256:ac0f3b379c94acc2f7dce5f5f0b24d44fa1cc6a509717ef83dfee07450c2117c \ + --hash=sha256:af2a2a8c7c74de0559e0c368d94c8def9e16c58faaee33a0bf081057c4227e3b \ + --hash=sha256:af98ad5ed9d6daaca956201e00bb429a7eb2b080426686f70a20353e0f9839f5 \ + --hash=sha256:afdf43b72ef3876c1fe66423b91466e37877c9e81e8cec70542b7e8525b9d1b7 \ + --hash=sha256:b88841e654f09732804809e435b3e005a929ffd9998b872b7b213957b8759cb8 \ + --hash=sha256:bb2fc905bbf4e6b7f40806ea79e31515abf6349594cdf0adf27c4215f0463204 \ + --hash=sha256:bb4ffe96aa663cee727659db5a2afeb38c95f8677b747d447b90d6d4874ea2c5 \ + --hash=sha256:bc0b0ac781d489304b741269857f1f8338b7a26b1b89c06c0344658001ec0035 \ + --hash=sha256:bf1bd822ec4e387ed245bed0d71151582cf7be9e5309bc4145eefe36083d5878 \ + --hash=sha256:c19cd6d025c1673f22afcd22c7df8a662d779e05d8e3fa6820c22afb895b0206 \ + --hash=sha256:c3305c38a2fa21a4254f2ace7dd9ef5fc569c9a558b66e7017650b3d637fb95e \ + --hash=sha256:c85d54e7e8a2ca932fe8399301af9b8d5907ea2a455ffaff6e7d1208db83b943 \ + --hash=sha256:ca64d9f1f384f151b9511bec01126072acd2f313439f8ed015a22d8790aab6fa \ + --hash=sha256:cce2bc991293f15cc4084ca116827b5900c5f34e1a54dfe83f10ab5c43162eb7 \ + --hash=sha256:d6276d78f6fca7d0ac066d5da4165c5acd07829e8305c2cb900b738fb3a75a72 \ + --hash=sha256:d93db87adb6b1c1b408dce4763314b55d76a9f589e96783a84ac9e7689e48bdf \ + --hash=sha256:db5f8394e17f877a625b257f2ba0ce8e728a499c2c1579ad66220272cd3df510 \ + --hash=sha256:db76506aa5416081f3e8974ae0f7965c58ada0bb0ef7339ac86099588dbb20d3 \ + --hash=sha256:dba2edfb054f6d4a08df9d1637c39a5aa3865bca6617c13c86be21e45658a59c \ + --hash=sha256:dcf4bc2aab4e16b1c4c0c2005918f23a7dd5d7821ddae82caed9e3342dc2fcce \ + --hash=sha256:e1fa594c887365b69745f25a416806e61085dd07b94c9eae68a6e20730629b23 \ + --hash=sha256:e6c52d3307824ff93b39efd99e4185d557db40bd841452abfb32e5d9151ca162 \ + --hash=sha256:eb57acff4a74246ae513c142d4b36e18c389c3aed8661914a53f7cd0071031b2 \ + --hash=sha256:f80bd9f9633eafc73d0a913ba2645c96ba58bba1befc30590f7c0fbfde59d865 \ + --hash=sha256:f8475460aa33ee28ac896ab1156d0bb3b6c639f7f8383c2677d3359eb35f8205 \ + --hash=sha256:fb2bde05838fffae1a1bf75e5d411a6cac3e4e9bb97e6640fed8cd47888b33f0 \ + --hash=sha256:fb9d92ecfe2d5b494367c67f7446f8b75b68d8d0c8cf3bc3e6997478be25d9e2 \ + --hash=sha256:fd3d72233eb8b48acc94fa57d44e2d32ce8e7abed02882ccb6d855ccc4ed33ec + # via docstrfmt +cryptography==50.0.2 \ + --hash=sha256:0ddc924c04591c2811ca024d62ecad4f7f6f08af8939c211438f48a16bd23602 \ + --hash=sha256:0ec5f09541743261e66e291b4a0cbf0fb2997aeaab6d9e9c740b9dba1b58d1c2 \ + --hash=sha256:0ecbc5652bdb6fc9eaf89a7d196e20941adfe812f43bc4ca05d9150496821047 \ + --hash=sha256:1981f1db4630889b9ef7803fadef12b056f428cb6b85c27ba57b774793b6093c \ + --hash=sha256:1ba34f04897fcdaa73f74145c25f3ec146fbd56593853e88adc2e811303c5f42 \ + --hash=sha256:241449bf940a5d27309bd317e6f9a2af6932113818bb2b8f5c59ddc7ef16da18 \ + --hash=sha256:25784ce8b9621c90c643efb9e1e2162ab3b0224cae446ad5e70e7fcb1ce18b51 \ + --hash=sha256:3dc4fd8058cea1644971207d530e1a03a184a805ffc8ebdddf0599d78a331b81 \ + --hash=sha256:4061c0079120205fb760c58acab6443e217307dcf05e3702cf970e0689972856 \ + --hash=sha256:4a20ce1e5cb4284a86692fdcba7cb8754185c6b2e5c56fcef3751cf451d3cdc2 \ + --hash=sha256:4e81d95e5bafc2d6e34e4bed780e53e4d5b9a2f928573428aa4d35fbec1eb0de \ + --hash=sha256:58a0c478eeca76fe5e07993c5a0703def34a6dc6a0cda4f5564639b33112ffe7 \ + --hash=sha256:58ddb5a8e3179d12f19e4ea34d2d32e9d63a4baa142c875c1eb59f41b7243acd \ + --hash=sha256:630ebfea3bf689d075f82316324ff7433dc447fe6bc1bfc76524b74b4a9567d2 \ + --hash=sha256:6f8700550aa1474a91e5dc07049c46f98b423b5b1ddd0483e0b51362eeeaf5be \ + --hash=sha256:78198641e5be9521beea5aa782bb551a58068d10e6eb04c9c680c1b69f2e7d45 \ + --hash=sha256:79def8d059362e7831389ed3be0ecdf58a89386e1271e35dd9f5af84e81bffd0 \ + --hash=sha256:7a8701d6b584d76e909e3d305b7d126b41439876a5aaf76cddc67fc230eafa2e \ + --hash=sha256:7afa5a6602a9f29af1f3a2965f831bae7c9d5d597b7cbb716d41ab3b7d89879c \ + --hash=sha256:7b46165bb56eb4704e2eaaf86f3c940d19154535d9b0ca7d6d590b04060e00d5 \ + --hash=sha256:7b75de3c8b3be1cdb1052747c929440c3eea46c1bc2cb8a6e3a48388e9b7b452 \ + --hash=sha256:7c6d0330c472d96f6a6afe24d80dfdf15176c33096f0a4397ae4c60f3dd3be48 \ + --hash=sha256:828d49b0ff5a0e3975865571c5d91dbbdd0d38d8289b249a163e9425413a5e05 \ + --hash=sha256:84f964e537f916e2cc85199e5a88742e964939b575ac8598b3f9d6cc416cdaf1 \ + --hash=sha256:85d0d9a31b9098e98534226d5686b47264b95e62ce459dc2e62fdfc809f9fe93 \ + --hash=sha256:87e9ce85beb6b328ba370cc6e6aea483c92617b4c95b1d33a49297eb662bfb04 \ + --hash=sha256:8c71ba2cd31fc93748c38e1b613200ff1c2665cbfd5341fe3a61cfde35a1430e \ + --hash=sha256:92e665960f25fcdc73725b9cec7a3824f279ba97a98653afe9ffac2e43668f67 \ + --hash=sha256:94e5e9f108ee10471288214d3d233fbfbb492840a8457eb85178d643ddeb32c7 \ + --hash=sha256:9c8402a82ea0dc4ceeab793db05f0fafa8ca139ca34fcde5df0f596103c74107 \ + --hash=sha256:9dab55f57c74c3cad24c323bacbbd04be4705ba6eb0d92e920b1fc4837ed5079 \ + --hash=sha256:a582ab2ae1d34f67112cadc86702774c9ea4374df6bca6afe672817203c99134 \ + --hash=sha256:a6557e5f38e065ca9fbdaf7cfc7435ecb1d113aa81a022d1b51921ee7432e227 \ + --hash=sha256:a9f7355e6fab51f6c369b86fb7571cffa05edee2c2121e0380a37fb9ac1cd5c1 \ + --hash=sha256:ab50ee449bf968271e820086f10a33d101dd060370abc10bcd22279be2656539 \ + --hash=sha256:ac9ed99d81760c62fe89d5f0815cdfa1ba9a35141cf30f1c2d044f04b4803d2e \ + --hash=sha256:b13478603dcd0a2479ff8e87e2c19a7d525734686fe3c49542472293a204212d \ + --hash=sha256:c423ab384a46c4dff7217b2ea5ba2e11cffdeab6441acd04cf65a369caf0366c \ + --hash=sha256:c5e67125c7dca78d199ec4e116aa93dbb83494808ecbb8211a2cb09b1bf41dbd \ + --hash=sha256:c71be1cbfa5cd9a41ee452acf1eccd82b2c05950358b106ec8ceb83411d1a020 \ + --hash=sha256:cbc8738fd8526d80f35cb3a40d41f41a2e7030bb3b18b09a6778ef63d291c2fd \ + --hash=sha256:ce47f66801c20ec6c6632453bb5960fe38939e9306970b48b3a5a26de7745d94 \ + --hash=sha256:d370b8d1dfcdf7130178137f6fbee6140774a1acc6cacefc4b42643ec11d0a3a \ + --hash=sha256:d38cdff612d06fa6a32840d5e1b1f7a27cee4a349aa9085d94a67789d6bfd408 \ + --hash=sha256:d8947001be83df1394050758ce0e745dd74fb134eef0a4b5124208dfc3a68c37 \ + --hash=sha256:deb9fde5c60e437ee4821bc9bc39ff31b42135c27e1dc61ef0a629389c1de62e \ + --hash=sha256:dfe9763530994147d9af1def057a5b9658b00e8f8fe8743d144d1e0911c2e454 \ + --hash=sha256:e105ab60406787da31fccc883fc0f733af1efd78f0136a4599692c4083a73d0c \ + --hash=sha256:e275096ea1e60cc595cda2836fd4a6c725d1125108b868be17f53684d164e2cc \ + --hash=sha256:edc3342adf8f697fc5f59c887a304356f147b397809440ed64e2fa6af2f50f37 \ + --hash=sha256:ee247f5c245c9a2fe7c8e2214e295918838e44e00a45a6718451e4004219e767 \ + --hash=sha256:eef4c2f3423810b3070ab391f85436d2f8bbfcb286ac15cbc73190b3563b1f1a \ + --hash=sha256:f21e8a22c8605750c7af886bab299a363721264061b4ac0a30efb73cfd58efc5 \ + --hash=sha256:f265528741e048bce55c3463ed721fb0aa45a5888d8add8cfeccb3035451bbdc \ + --hash=sha256:f2f9bd7f90c64fe89253f0a2c05e3c4856072660429ce8831b4235bf29403a67 \ + --hash=sha256:f785f6161f202ab04d8ca194158968798e480ca058943907972da5f12e2881e8 \ + --hash=sha256:f9f6143a8c75945eb960d9eb98905a441394abfa24afaae239d514ffb2586480 \ + --hash=sha256:fa8f5efb344d6908a1ce62f4a24e2e5780f825d6f53f5f50ec5ffacac72936cb \ + --hash=sha256:fdd28f912fccfec1846a94e2e1e8f9b0012f557f0c46fe4f3eb0d7a87afcf90b + # via + # pyopenssl + # pypi-attestations + # rfc3161-client + # sigstore +cyclonedx-python-lib==11.12.0 \ + --hash=sha256:0e807521a921a5c3cb8ce1153f8a61d29eedfe76a46aac2796b7c6b573391a54 \ + --hash=sha256:16767c4039de90c04e9f03348f8f0ed4b8ff842eaa7eefcad3a95685f970dacf + # via virtualenv (pyproject.toml:release) +defusedxml==0.7.1 \ + --hash=sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69 \ + --hash=sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61 + # via py-serializable +distlib==0.4.3 \ + --hash=sha256:4b0ce306c966eb73bc3a7b6abad017c556dadd92c44701562cd528ac7fde4d5b \ + --hash=sha256:f152097224a0ae24be5a0f6bae1b9359af82133bce63f98a95f86cae1aede9ed + # via virtualenv +dnspython==2.8.0 \ + --hash=sha256:01d9bbc4a2d76bf0db7c1f729812ded6d912bd318d3b1cf81d30c0f845dbf3af \ + --hash=sha256:181d3c6996452cb1189c4046c61599b84a5a86e099562ffde77d26984ff26d0f + # via email-validator +docstrfmt==2.2.1 \ + --hash=sha256:9f35b62e746962efe32f7af27b3bf69e104a4e69827a49a9f488da770271bd0e \ + --hash=sha256:a0a9962dbd64684b9bea2af582c631ae174d06850cd960eb37221dd6a0d6cdc4 + # via virtualenv (pyproject.toml:release) +docutils==0.22.4 \ + --hash=sha256:4db53b1fde9abecbb74d91230d32ab626d94f6badfc575d6db9194a49df29968 \ + --hash=sha256:d0013f540772d1420576855455d050a2180186c91c15779301ac2ccb3eeb68de + # via + # docstrfmt + # docutils-stubs + # sphinx +docutils-stubs==0.0.22 \ + --hash=sha256:157807309de24e8c96af9a13afe207410f1fc6e5aab5d974fd6b9191f04de327 \ + --hash=sha256:1736d9650cfc20cff8c72582806c33a5c642694e2df9e430717e7da7e73efbdf + # via docstrfmt +email-validator==2.3.0 \ + --hash=sha256:80f13f623413e6b197ae73bb10bf4eb0908faf509ad8362c5edeb0be7fd450b4 \ + --hash=sha256:9fc05c37f2f6cf439ff414f8fc46d917929974a82244c20eb10231ba60c54426 + # via pydantic +filelock==4.0.9 \ + --hash=sha256:635e7d67fa92654eed444e75e9ca18426d34e77ad9c469bf4373f75a932f7b22 \ + --hash=sha256:9287fd61b99a806e5202be29a83034c0808a1b9830e820537c2f5773981f7eeb + # via + # python-discovery + # tox + # virtualenv +fqdn==1.6.0 \ + --hash=sha256:6c793c312ccfa29981e1135b3ef5e1277ebe639731a6b0f7350d90e96e63f0b4 \ + --hash=sha256:e39bf62e2a9481aa2cb780554fdfd944dd8ee5f230eb831684d539cbafed8de2 + # via jsonschema +gitdb==4.0.12 \ + --hash=sha256:5ef71f855d191a3326fcfbc0d5da835f26b13fbcba60c32c21091c349ffdb571 \ + --hash=sha256:67073e15955400952c6565cc3e707c554a4eea2e428946f7a4c162fab9bd9bcf + # via gitpython +gitpython==3.2.0 \ + --hash=sha256:bd70c5ec05cd2b797423e7eb312147d2458d3cca92085888fba2213f85905537 \ + --hash=sha256:fb92310af6844d96adc95ca066ed2e617c00e1dbd146a326626c81e72e18cc2e + # via virtualenv (pyproject.toml:release) +hatch-vcs==0.5.0 \ + --hash=sha256:0395fa126940340215090c344a2bf4e2a77bcbe7daab16f41b37b98c95809ff9 \ + --hash=sha256:b49677dbdc597460cc22d01b27ab3696f5e16a21ecf2700fb01bc28e1f2a04a7 + # via virtualenv (pyproject.toml:release) +hatchling==1.32.4 \ + --hash=sha256:08ecf7548fb48205e7f213d70c71e67b8271b7242093dc3f1da578b42c734a2c \ + --hash=sha256:c4468f73144c054d2aab4ef0f0378c43b9878bf07f8ffd6b79690e970d375f07 + # via + # virtualenv (pyproject.toml:release) + # hatch-vcs +id==1.6.1 \ + --hash=sha256:d0732d624fb46fd4e7bc4e5152f00214450953b9e772c182c1c22964def1a069 \ + --hash=sha256:f5ec41ed2629a508f5d0988eda142e190c9c6da971100612c4de9ad9f9b237ca + # via sigstore +idna==3.20 \ + --hash=sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44 \ + --hash=sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c + # via + # email-validator + # jsonschema + # requests +imagesize==2.0.1 \ + --hash=sha256:b2ba6a4dea487a7ebcd53248d3476aca449d30db12a2dde5e0c5ca9624fd77e5 \ + --hash=sha256:ea0c9a0384df69ed86a943a15cde37d0360b82491b3910dc2215e202e62b5b02 + # via sphinx +isoduration==20.11.0 \ + --hash=sha256:ac2f9015137935279eac671f94f89eb00584f940f5dc49462a0c4ee692ba1bd9 \ + --hash=sha256:b2904c2a4228c3d44f409c8ae8e2370eb21a26f7ac2ec5446df141dde3452042 + # via jsonschema +jinja2==3.1.6 \ + --hash=sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d \ + --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 + # via + # sphinx + # towncrier +jsonpointer==3.1.1 \ + --hash=sha256:0b801c7db33a904024f6004d526dcc53bbb8a4a0f4e32bfd10beadf60adf1900 \ + --hash=sha256:8ff8b95779d071ba472cf5bc913028df06031797532f08a7d5b602d8b2a488ca + # via jsonschema +jsonschema==4.26.0 \ + --hash=sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326 \ + --hash=sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce + # via cyclonedx-python-lib +jsonschema-specifications==2025.9.1 \ + --hash=sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe \ + --hash=sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d + # via jsonschema +lark==1.3.1 \ + --hash=sha256:b426a7a6d6d53189d318f2b6236ab5d6429eaf09259f1ca33eb716eed10d2905 \ + --hash=sha256:c629b661023a014c37da873b4ff58a817398d12635d3bbb2c5a03be7fe5d1e12 + # via rfc3987-syntax +libcst==1.9.0 \ + --hash=sha256:02ee2dbdb5c218116f16a021350fc267a14be205b50d81c33f5d73857ab6fbf7 \ + --hash=sha256:03165a264653bb77f6a11b412ae09c08bdb0c25864f3b8d42b816ac64b9d4b9e \ + --hash=sha256:087b58a9afe076bb08e2d726478e1f16cb928d67ffa9092817e033c335de522a \ + --hash=sha256:0c7b548512db25af9c2997a95fa731bd6b6928ecbad6c0915d7482d8bb42d34f \ + --hash=sha256:0ef45740663d0e6d8159b9eb6db436c4eb8dd0808c30812501542d65cd44c030 \ + --hash=sha256:107593af46945593e7825821793393262bc4fa1d3ea24c3ed487b61269bbbdf8 \ + --hash=sha256:1260b5d070a3324447a53a00387225a55472a62077ce01922d30a2a78cc4b138 \ + --hash=sha256:1879e5bd97f124acc787ae73cf52975cea46ac0282db5e8d9296173fbe87fa21 \ + --hash=sha256:2b150c4f298fe54eb0fe73abf71f57db74d070796140a8c206c9615b6861f01e \ + --hash=sha256:32395244edfe6538e0ea2bf82051d60103d3f54861274805c4fb3745efb70a85 \ + --hash=sha256:329ffda6b0ef708ff22876f982ec886a509c8f73ac1ccb0190693d35cb09aa0f \ + --hash=sha256:3cdbafbc992b38ea005ef37e20243c4177879020c82d414789fbb7fa22d6d3a9 \ + --hash=sha256:3e5b684191a0462b2d40a73261ea7f4da6a49e7a030b7e0fceca46bebb51dfe5 \ + --hash=sha256:3f735e3acb3afc75d50803fde1598b467bf05d12d6854025df4398aebd75a14c \ + --hash=sha256:4090105fa4ad114766d0f8b34418d9a3b359bc8645da860a447e19770b5705d1 \ + --hash=sha256:444e84c76cd035cd2fe136838c1a524d34b08216521f6f5093df8a6f6cfa5799 \ + --hash=sha256:45808c03528b3ad40b14095348a918e08d98c47b4a125d631cb78e817c0a5b16 \ + --hash=sha256:496c24e0d3240bc7da45dae543aff3f5b6509978c39262d6b84ed2fb999dded2 \ + --hash=sha256:497d5329345f1f5df84e41b0bbd00204b64a2fd30dfe3cfaeaebca633a31e877 \ + --hash=sha256:4adf97bb1aff8039b0bd4991e2f838be6e4fad552821b26b71a5d1a653c2582f \ + --hash=sha256:4d382fba04077eb556a1ea4295a4482e192aa93639c5a07ba0885841965ea0c0 \ + --hash=sha256:4fb016ad6ec8485991770dcff3fe072b0296eff6fefe7b3b4e28fb919dea3def \ + --hash=sha256:50ab94bb2524b419056d4003032b8c67102ac800f8d85b3c4260a01746d94dbb \ + --hash=sha256:50b913e37187f00a6fb88962009364e1cf1b1284aa1762304f34b52b972f2218 \ + --hash=sha256:5302fc698ef863e1a2fb540e95793b3d5a96abc89aa6423ee9c389d7d7ec5eab \ + --hash=sha256:568288cdbfe3b4ca3ae4852cb0a439ff053dd54c841bc4995bf2b71238b5de40 \ + --hash=sha256:5705f664266742b8d9323db714f58cf06a6cde739eb09fd9ca31cd233f9806e5 \ + --hash=sha256:5891ce9cff815077614f509e3a23888c5ddb8081d6188e8ba1172c0ccc369022 \ + --hash=sha256:611cebd3bbc2014576f4dcc7b845b3c594c96ddc287a3db9b78f22eff156a7d3 \ + --hash=sha256:6e50576bad7d56459d9792cd0b0dfe5469dad13646e9a9c0a8b1ba20b269f332 \ + --hash=sha256:77d9303572cf0fbd0a2eb71aff167a4246541f43bf58b781b35b9d0bbab80ddc \ + --hash=sha256:7acfd18adcdd32dcf41ce676cf121002afcbe9ad2c72dc0c18d78465beedc228 \ + --hash=sha256:7dcddfb4905c5ef90a6fdc58dccbc049faf0056b68161042fc1e0526b20cba05 \ + --hash=sha256:8504b422c95676a8c27b517e1ac01413ece91bf356865c587ca9bdcd5708a2f7 \ + --hash=sha256:86361e2b426bd1b403e52375703c8b764e226e571adcb30442510710681edbf0 \ + --hash=sha256:8930971d2299b7006bb5d10c10038f44efb6da89d5bca2823595e31a9cdb96af \ + --hash=sha256:8bc5351d92ca6ac1cc32e097700e1161ad1ceaa4d9b2cca5abadb1e94576b325 \ + --hash=sha256:8c14abe844bec8b021111b3985474e49f5af799c020e8e40dedcac87c97a40c5 \ + --hash=sha256:8d731abe1307720ea1a52d447555e8443a6d130e0e520243c0634a58f6edbc9d \ + --hash=sha256:8f0dd08a5773d7051e105250b2a2c73d8065ea9b2d68e7e1bdc5244660e75f93 \ + --hash=sha256:951f13c193e590fc88c9582a5341a130fe53ecece51043cff4bd54e25a5e022d \ + --hash=sha256:9f92c75283030fd58fb7d6e560168a00381e0e3368ec8a026a3ec8a828a05f93 \ + --hash=sha256:a2faaf92500d0226358125630f5aab4758e8aad3f2d70a10892ec3c700781a54 \ + --hash=sha256:a5068bf6114f6f4d79af7a6c80a28d1deb50441150ea1db13b5458ab34bec159 \ + --hash=sha256:a621e261990148c1cfbe26c1798bd2375c131cf358a44a7a4659fc41c1a336e1 \ + --hash=sha256:a6615b036694f6906e30dc018b5b34898cff7ff760ab549f9dea1ada4f53495b \ + --hash=sha256:ae8b2abe4d5f52bfcad749350e4483976b7b5142f765ebab69959f2b4f99641c \ + --hash=sha256:b18c9d7316f96ab255a9a127289af0b9b7d1ebf8929e81eefdeccd951f4ba58e \ + --hash=sha256:b4fa00cb71ffbda877dc4010ae8f3330ca605af859ddc6444bea96c4f294d648 \ + --hash=sha256:b755ed4a4bc2faee849b54820023137d60d4c199e0a0822f0ff0c1bc49e49b48 \ + --hash=sha256:b8b9df30f524317b097dc53065b25dda33d6a4cc3c7c8bf4fc83ca7559c58cc0 \ + --hash=sha256:bb5d0946f2b4c6711b5d69fe4f833b364f9e7a1a2b08f88b98619dc18975099a \ + --hash=sha256:bc6aa3e7f21315dbe9868c0d13ffaac309514e8c816320c5f91230fa5b21df92 \ + --hash=sha256:bcb9f9d4fcfe2ec7a40d2c26e03a538d5d9dc189c38551eb3f94ab661afee7c0 \ + --hash=sha256:cc84aee3c293a012db722aa2d6ea87be524646852684a5a4e54c1fb98ea9072f \ + --hash=sha256:cd1a3500c41784075c4946a995d5ad89f68fa0d226b63ff3c4d78f6ea6dd23e5 \ + --hash=sha256:dacc583bcd8db56e457ae559b910673c6b5df66f694d1f1154ce0b3982158d17 \ + --hash=sha256:e465a7bc9c2b9533eb9e06d2391f8819f811b5112919d4064c9fb8565aaafa08 \ + --hash=sha256:e8d671c39a431c309476099b8ec811e412503ec0f4465f6fc907cb51c70e8e6b \ + --hash=sha256:ea490fa8540503db5f321f0268becab46eb50f710e8cec8041e241fd66f6874f \ + --hash=sha256:eccf4c57d273cdd3fe1c67b72cf9bb1bbd4547aa011824e96ccf5b7136057aa4 \ + --hash=sha256:f101e0bf8638eba80694ecc3c3abb5309eb488f5271427536cb379c5c1e0ba9e \ + --hash=sha256:f6248cb07444ab9a6733a855737a9febed8b9adca51347019348b09a3ac7dfe9 \ + --hash=sha256:fceeaf1a1fd186557973904cfd2bb3c05ad835b01d7d04e9ec1df463c0add966 + # via docstrfmt +license-expression==30.4.4 \ + --hash=sha256:421788fdcadb41f049d2dc934ce666626265aeccefddd25e162a26f23bcbf8a4 \ + --hash=sha256:73448f0aacd8d0808895bdc4b2c8e01a8d67646e4188f887375398c761f340fd + # via + # cyclonedx-python-lib + # spdx-tools +markdown-it-py==4.2.0 \ + --hash=sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49 \ + --hash=sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a + # via rich +markupsafe==3.0.3 \ + --hash=sha256:0303439a41979d9e74d18ff5e2dd8c43ed6c6001fd40e5bf2e43f7bd9bbc523f \ + --hash=sha256:068f375c472b3e7acbe2d5318dea141359e6900156b5b2ba06a30b169086b91a \ + --hash=sha256:0bf2a864d67e76e5c9a34dc26ec616a66b9888e25e7b9460e1c76d3293bd9dbf \ + --hash=sha256:0db14f5dafddbb6d9208827849fad01f1a2609380add406671a26386cdf15a19 \ + --hash=sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf \ + --hash=sha256:0f4b68347f8c5eab4a13419215bdfd7f8c9b19f2b25520968adfad23eb0ce60c \ + --hash=sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175 \ + --hash=sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219 \ + --hash=sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb \ + --hash=sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6 \ + --hash=sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab \ + --hash=sha256:15d939a21d546304880945ca1ecb8a039db6b4dc49b2c5a400387cdae6a62e26 \ + --hash=sha256:177b5253b2834fe3678cb4a5f0059808258584c559193998be2601324fdeafb1 \ + --hash=sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce \ + --hash=sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218 \ + --hash=sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634 \ + --hash=sha256:1ba88449deb3de88bd40044603fafffb7bc2b055d626a330323a9ed736661695 \ + --hash=sha256:1cc7ea17a6824959616c525620e387f6dd30fec8cb44f649e31712db02123dad \ + --hash=sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73 \ + --hash=sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c \ + --hash=sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe \ + --hash=sha256:2a15a08b17dd94c53a1da0438822d70ebcd13f8c3a95abe3a9ef9f11a94830aa \ + --hash=sha256:2f981d352f04553a7171b8e44369f2af4055f888dfb147d55e42d29e29e74559 \ + --hash=sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa \ + --hash=sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37 \ + --hash=sha256:3537e01efc9d4dccdf77221fb1cb3b8e1a38d5428920e0657ce299b20324d758 \ + --hash=sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f \ + --hash=sha256:38664109c14ffc9e7437e86b4dceb442b0096dfe3541d7864d9cbe1da4cf36c8 \ + --hash=sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d \ + --hash=sha256:3b562dd9e9ea93f13d53989d23a7e775fdfd1066c33494ff43f5418bc8c58a5c \ + --hash=sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97 \ + --hash=sha256:4bd4cd07944443f5a265608cc6aab442e4f74dff8088b0dfc8238647b8f6ae9a \ + --hash=sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19 \ + --hash=sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9 \ + --hash=sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9 \ + --hash=sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc \ + --hash=sha256:591ae9f2a647529ca990bc681daebdd52c8791ff06c2bfa05b65163e28102ef2 \ + --hash=sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4 \ + --hash=sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354 \ + --hash=sha256:6b5420a1d9450023228968e7e6a9ce57f65d148ab56d2313fcd589eee96a7a50 \ + --hash=sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698 \ + --hash=sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9 \ + --hash=sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b \ + --hash=sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc \ + --hash=sha256:7be7b61bb172e1ed687f1754f8e7484f1c8019780f6f6b0786e76bb01c2ae115 \ + --hash=sha256:7c3fb7d25180895632e5d3148dbdc29ea38ccb7fd210aa27acbd1201a1902c6e \ + --hash=sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485 \ + --hash=sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f \ + --hash=sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12 \ + --hash=sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025 \ + --hash=sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009 \ + --hash=sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d \ + --hash=sha256:949b8d66bc381ee8b007cd945914c721d9aba8e27f71959d750a46f7c282b20b \ + --hash=sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a \ + --hash=sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5 \ + --hash=sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f \ + --hash=sha256:a320721ab5a1aba0a233739394eb907f8c8da5c98c9181d1161e77a0c8e36f2d \ + --hash=sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1 \ + --hash=sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287 \ + --hash=sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6 \ + --hash=sha256:bc51efed119bc9cfdf792cdeaa4d67e8f6fcccab66ed4bfdd6bde3e59bfcbb2f \ + --hash=sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581 \ + --hash=sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed \ + --hash=sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b \ + --hash=sha256:c0c0b3ade1c0b13b936d7970b1d37a57acde9199dc2aecc4c336773e1d86049c \ + --hash=sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026 \ + --hash=sha256:c4ffb7ebf07cfe8931028e3e4c85f0357459a3f9f9490886198848f4fa002ec8 \ + --hash=sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676 \ + --hash=sha256:d2ee202e79d8ed691ceebae8e0486bd9a2cd4794cec4824e1c99b6f5009502f6 \ + --hash=sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e \ + --hash=sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d \ + --hash=sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d \ + --hash=sha256:de8a88e63464af587c950061a5e6a67d3632e36df62b986892331d4620a35c01 \ + --hash=sha256:df2449253ef108a379b8b5d6b43f4b1a8e81a061d6537becd5582fba5f9196d7 \ + --hash=sha256:e1c1493fb6e50ab01d20a22826e57520f1284df32f2d8601fdd90b6304601419 \ + --hash=sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795 \ + --hash=sha256:e2103a929dfa2fcaf9bb4e7c091983a49c9ac3b19c9061b6d5427dd7d14d81a1 \ + --hash=sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5 \ + --hash=sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d \ + --hash=sha256:e8fc20152abba6b83724d7ff268c249fa196d8259ff481f3b1476383f8f24e42 \ + --hash=sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe \ + --hash=sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda \ + --hash=sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e \ + --hash=sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737 \ + --hash=sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523 \ + --hash=sha256:f42d0984e947b8adf7dd6dde396e720934d12c506ce84eea8476409563607591 \ + --hash=sha256:f71a396b3bf33ecaa1626c255855702aca4d3d9fea5e051b41ac59a9c1c41edc \ + --hash=sha256:f9e130248f4462aaa8e2552d547f36ddadbeaa573879158d721bbd33dfe4743a \ + --hash=sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50 + # via jinja2 +mdurl==0.1.2 \ + --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \ + --hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba + # via markdown-it-py +mypy-extensions==1.1.0 \ + --hash=sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505 \ + --hash=sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558 + # via black +packageurl-python==0.17.6 \ + --hash=sha256:1252ce3a102372ca6f86eb968e16f9014c4ba511c5c37d95a7f023e2ca6e5c25 \ + --hash=sha256:31a85c2717bc41dd818f3c62908685ff9eebcb68588213745b14a6ee9e7df7c9 + # via cyclonedx-python-lib +packaging==26.3 \ + --hash=sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79 \ + --hash=sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c + # via + # virtualenv (pyproject.toml:release) + # black + # hatchling + # pypi-attestations + # pyproject-api + # setuptools-scm + # sphinx + # tox + # tox-uv-bare + # vcs-versioning + # virtualenv +pathspec==1.1.1 \ + --hash=sha256:17db5ecd524104a120e173814c90367a96a98d07c45b2e10c2f3919fff91bf5a \ + --hash=sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189 + # via + # black + # hatchling +platformdirs==4.12.2 \ + --hash=sha256:29dbf06d96c500bc6bdbce75fb0a14d63279c93b1842f97e72a135b33e856983 \ + --hash=sha256:eab5f70271a490ef74618bb314fbb86e3c7e82fa3b9c922c2ea0e0a1a155d329 + # via + # black + # docstrfmt + # sigstore + # tox + # virtualenv +pluggy==1.6.0 \ + --hash=sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3 \ + --hash=sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746 + # via + # hatchling + # tox +ply==3.11 \ + --hash=sha256:00c7c1aaa88358b9c765b6d3000c6eec0ba42abca5351b095321aef446081da3 \ + --hash=sha256:096f9b8350b65ebd2fd1346b12452efe5b9607f7482813ffca50c22722a807ce + # via spdx-tools +py-serializable==2.1.0 \ + --hash=sha256:9d5db56154a867a9b897c0163b33a793c804c80cee984116d02d49e4578fc103 \ + --hash=sha256:b56d5d686b5a03ba4f4db5e769dc32336e142fc3bd4d68a8c25579ebb0a67304 + # via cyclonedx-python-lib +pyasn1==0.6.4 \ + --hash=sha256:9c447d8431c947fe4c8febc4ed9e760bc29011a5b01e5c74b67025bd9fb8ce81 \ + --hash=sha256:deda9277cfd454080ec40b207fb6df82206a3a2688735233cdcd8d3d565f088b + # via + # pypi-attestations + # sigstore +pycparser==3.0 ; implementation_name != 'PyPy' and platform_python_implementation != 'PyPy' \ + --hash=sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29 \ + --hash=sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992 + # via cffi +pydantic==2.14.1 \ + --hash=sha256:9195d967ec791692a04438115466764fb8b9a27b31f14a760437694f40d6b454 \ + --hash=sha256:94f478203dd03404682a1ada216965651dd74b1d2d5ffd62e00e0837caab5c26 + # via + # pypi-attestations + # sigstore + # sigstore-models + # sigstore-rekor-types +pydantic-core==2.50.1 \ + --hash=sha256:0036473f5583e6a60e50b8b21651511564277a3f05cc5dab8cf579f552cd5f6c \ + --hash=sha256:0048b6dddc8ef4b64fccaad878bd143b0c3882ea9936279dc11d613f6b7dd1bc \ + --hash=sha256:009634b83993777ddcd69cad0ffcace43dabde692109528e35f0fde91e386a8b \ + --hash=sha256:028e2f212273d4a39b1ec1e0de8166b1165a65fc0f1111452a9d94fc7c625c63 \ + --hash=sha256:062e891facce5ca296a1c37098e5e466780457f86413894b399f0cf22934f769 \ + --hash=sha256:06e01fbbfdb9be777b316a71b6c49efaf4a08b615d0a98d678cda3023f79d019 \ + --hash=sha256:06ead20d39ffd6f2f6f2a8f8a6de67ff8bb1b4f14a8a30e058502514ee2ac685 \ + --hash=sha256:0b3a6f334c6a2345ca15318ff894502a90012536404b37c844a976c76c846e0b \ + --hash=sha256:0c003c3b7f49debb893d2d85ae099ac5959c9839e2f330fadb1fcdf7a6594482 \ + --hash=sha256:131059670f1d2444269b8585cb888963994871932447c08b39ac6a51fcfef658 \ + --hash=sha256:132529c83901437ff642f585216831bf5fd7a91df66829907e155192ead62498 \ + --hash=sha256:17e722e156d0444ecaefbe640bdb60928752bf2013e2b7a11cdb099aaae19bec \ + --hash=sha256:1a9006395dece0e32e704c315eff8a00bede494f6108546cfc5539c89fef4f9a \ + --hash=sha256:1c8632d4ac04e6f91128fca584b3a8a507d81604c24eeaaad00d4be42765c32b \ + --hash=sha256:1c96fd793b73d1b92e65570132505498fe7b21eaef73cdf74e67e5dfba7ac9e4 \ + --hash=sha256:1cf41f1ae3fa155cf167a72689ad044bcc1e3c97e064123677149bdfb5dafc4a \ + --hash=sha256:1deeacb112d14d3f4fcb16b165f7dbaf76c70ba6e82f37ba042bdab51970a0b8 \ + --hash=sha256:1ef800dd7d85bcdadf4c3076e4c94e43939493558a3b69a1ea830c706d4617bb \ + --hash=sha256:1fa4c8bc12c1354c5550c0c35c1852c8c1901e89e06561724e03f8d0342e1f87 \ + --hash=sha256:2005207aafe1231315718bf6ed5d064a7300fb4772754af35ee72fc68159492e \ + --hash=sha256:23923ab9292c40da026330b1ecf4dc2618c8e86e0422e5d1fbf50d94d64ca4f8 \ + --hash=sha256:23edad659e8dbd8ca7e4e877fe6c81573abbdf215bd25a68b53e1272f58b80c7 \ + --hash=sha256:2ab756b72bd5054e4c7ef3ded331b35786cbd3cf931531a508f79a9537517064 \ + --hash=sha256:2cbd1b75b09e976ed0d6b6ca297675632ca35df86130088457cdc60ef36970ae \ + --hash=sha256:2cf91809d0721ab81592ba67bea7694821679c10b1a2e3c3460082b286c1918a \ + --hash=sha256:2df1ff41884de2bc4b307bafd7c40a691094fad2ff8e767e5b45a319257bcf4e \ + --hash=sha256:2eb75304506894a281d346220a4f7481a1b8729577c5ed2a05395991966a8396 \ + --hash=sha256:2eedf82ee4753cdab8e50044c6bd569577eebc3859b11fecf4eb9223761ff966 \ + --hash=sha256:30ddf019d082c117b5d309e5b86710c2a78909907ec1a9381feec3eec02eca0b \ + --hash=sha256:325c23f3e35cfbf0fe3486fa5f7260d1e45885173002d30a28ca019994124255 \ + --hash=sha256:32fad3a91e51b6d2039c572db04a5a873260b399f6bd62c3552671fa7a4a2899 \ + --hash=sha256:36c426eac0af8d1529ff8467e612b933346caec1fdc0d774f78f67a1a11e16c1 \ + --hash=sha256:3a5fce22f1e87d181e924e12da7d81cfe031fb3881a5ddf26ad28f141756ca43 \ + --hash=sha256:3aa9de446b793de2beb6fa2d9d0961803126c4e2a99c2f25ab59b9fd6ea125c0 \ + --hash=sha256:3e46a9eb0a0901dd6275e6b06ac3a464885ef350ec4121fe486869de8053e4bb \ + --hash=sha256:3fde4fdc6487a58d944ca87cf5adc95d5f266e872c19599f5f4c0a8a1b1f9f9f \ + --hash=sha256:409e0ea40ec30d9158f33574fd758e689f6045a0f2596701828c27816ca9687d \ + --hash=sha256:40f523349960fa30f3ea51404308ff50f9997a90df639590f47a057c1f32b415 \ + --hash=sha256:41bc8237121bd8dc8d888dfd6279fc166ffc88c1f1bf3a8bf00869680533ca4c \ + --hash=sha256:42b54c2c90ad348b5e3a85e03e715d572c1fde357ef104cdfe3b03b697a404ea \ + --hash=sha256:455a773617b5913bf5c20d0692e5787b119e52c4d40ea644ca31f5758fd31be2 \ + --hash=sha256:45b11cac094aa25725581d9304eee93c9028516b9ea80dd9e175e13a5a2c840e \ + --hash=sha256:45c6266d071c241f2a168d45bf8c54344f0effce35e7e6b73afdec11f3687568 \ + --hash=sha256:46b3301d3b5c886f77de7546e47274a5842c622ea2020b8c6524c6b66913b4a6 \ + --hash=sha256:476f6ed8e43cd1e0b460920e23571700872b284e77331cb30c4faf459cf48a4b \ + --hash=sha256:48569b0ade9edfbe065cad1d700175546592aebbb42f02adcebcc26e75b896fe \ + --hash=sha256:49c2cbb2397fe4d0987e84606e691af6cb87bc0ee1bd3e7b737f7e10b4c142f9 \ + --hash=sha256:4a53d13cdfbedbfa87f08b83c1a0a5efcc767d785a4b41934fa9cb672670493a \ + --hash=sha256:4be846f55c9477f5f3ddde8f2ce941137e16862a56d018ed885d422bb6ae02f2 \ + --hash=sha256:4df197990c15b5a37c5a277d131d9f2c67de6133f2e5dafd80d9bba4b99f46f9 \ + --hash=sha256:4e834f6a8e4ff772dcc34f58ef5504147a3ea5b0f4eeb13b0f8eb2ca75ac57f1 \ + --hash=sha256:57f51b31ff826e2859120cf4737c5a758a48d96f3e97da40ccee1796d58078ff \ + --hash=sha256:5958c72adb417c39b12ac87525ac60b0d73315fcdc59e21f44ee4a5e2512c9ef \ + --hash=sha256:5dfe41f232befddb9c4377f6cfc702b51595e2d78ed082672adf8758d2c4619f \ + --hash=sha256:5f3cae32fc46121f787cb2486de9cf95a8bf72aec5cc78f64c606fa1735a6ef5 \ + --hash=sha256:64f6047f62a6c5ae08d0a6afb035667aa2d97c3d20d69762e034c5ea144d92a5 \ + --hash=sha256:6a733778df2f7087ec1100ed0b41533e4f3001976e99570fa34f57c66e7f8e3e \ + --hash=sha256:6b20a4bffabdad0db2927ac034ae3b8a681b1f7a0182f3e60b479ad2fde21ebb \ + --hash=sha256:6dbcbee53bf17196a7f745aa9bf5a9603953a1e365b1f020be3207c676a3e7c4 \ + --hash=sha256:6ed4f3cef55164b026fefb41341b7754cc6b624c75dfe7142d2ecceb5ad21c87 \ + --hash=sha256:704075d10b74f2f3c6e15407c696d88701df35fc8953f434a431add0d0074db0 \ + --hash=sha256:739dc730e6be3bd5ec2f4ab5cfc7eb047cc45fc1497b3bafec74ff2ed07df597 \ + --hash=sha256:7456d699b13954e9c0164dcb267250a10ae0dfb03e6e26d6796ab0d46e189c84 \ + --hash=sha256:756d669f04e62ec4148ecfe22be6a4484d9b1181a6ef32e205ebfd200540858b \ + --hash=sha256:7689580e72a642ab5ec64d5f55b2e33636fa43b4ebe63c0c2c965ef307c7d1aa \ + --hash=sha256:76e2e83fa6ec8cdc972d438dafc2522b3a47bee4ec0ae668b29cfb1977ab5242 \ + --hash=sha256:7816e98acc08119dc0f340ab167048ecc54126316330c1f0caf7c6756c88e28f \ + --hash=sha256:79490e33c4c0fcb933bbbcfc3a62184d8803b99f535863dfbb925e1bcb6945ad \ + --hash=sha256:7f476456ac2bb0d937f75191494a09c83a30765fea4f70f3b404942fe25f6cdf \ + --hash=sha256:844b869f118e22a41a091bdcedda8a71bc1b0f62c38d1a0c3211cece47e1d8fc \ + --hash=sha256:84bc765b282a9d5b7fe0348b8648904f25a6a04b2139da52b1dd30c8ac3a2c8f \ + --hash=sha256:84f34323a61a365b4e9295de6028474754829aaddd59c7bf1a040e7487ef8f3c \ + --hash=sha256:8812592c85d0edf423f10eadcef42716d71e8219085ad9e85b775057b7306133 \ + --hash=sha256:88e492e8b9d0312e7dc13667c30222abf284dc3b79b5302b3607b41a5784ce61 \ + --hash=sha256:8a6791afa2245e6c6b180122d105941644f5bd410bb18623b408808cc41a3102 \ + --hash=sha256:8b4c3df25bd323bf1d36a648d563cf1fc69d717451569927151bdad7cad07a77 \ + --hash=sha256:8daa7ee75245d43ad7d747e5c9ecc1b1d06552f72b14887e9276f787d57375f4 \ + --hash=sha256:93ba4e9d8210d941c200431a56b2c0400b131865947903937ed3ec5404307d2e \ + --hash=sha256:94845ff54dc5193f228cab81b2662a04bfbb892e95bdc15edf7399000ce57d54 \ + --hash=sha256:94be440c03fede26969a5ce75468e0e6a9927a1b46d9b679ee8adc1b057b0350 \ + --hash=sha256:9572c1369e9c9da2d64a7b7992c786d90ff295abc93964cfe3125e4290768070 \ + --hash=sha256:983a662de2571cb2502fc8ff47b6770b03d025d2eb314c92f77b3f07c74720ed \ + --hash=sha256:992c3514ec891fa7858099183e4d64e6bd5a5d4ff452fae29df22faa77a006bb \ + --hash=sha256:99ba9bc2b8062ea0c326a990f7f00e6530c23579de66dd246e72c4cafef950a5 \ + --hash=sha256:9d1bed94af6a63835461f3cf7502058eb166c58c4778e11d0f433cfb1bd69e19 \ + --hash=sha256:9e4472072de0137ee0d8e72d6620e85939c271d2f90f6bbb4b15c24638b79f92 \ + --hash=sha256:a27c09d86600f1bf2fe3f37e1ae697faf3143931c09322cd799da94deee923b5 \ + --hash=sha256:a29a061fec0b4e2d714f277e70a3a18125ecff803f2fea6eade2f2e53711d112 \ + --hash=sha256:a3cda0e538208e5d722bbf3698b24f19c0a7d05bc8d5f8a7f9b121ea7fa243d9 \ + --hash=sha256:a44101320cfe99432db74237545a63057dc7a88dfe792cbcad0647f2af56cb81 \ + --hash=sha256:a4aaaa791bdae1c972a7e81765f4f3571c926b8e0b9b6e47346499fb80079665 \ + --hash=sha256:a51eee75939cf811ac09b278745a6cee7dc873ccfbc8b9af3cc88fe4b7ce25b5 \ + --hash=sha256:a7c58106de36ac6a56314182958de20db8d3a29dfd5db527192cc754e4f8e7fb \ + --hash=sha256:aa8224f10880d9bf1b5993988ba153d42a8b4f3f4f511f93b1f09c93ff613c72 \ + --hash=sha256:acbf31f37c53a5ac0c34706c80b4f5107ba20b05fdd3816124bf236ef0c57dd2 \ + --hash=sha256:adc06d218a1cadfd2ec4628424d7d79ce4eba69c2965e7e7b55106f0da5208c8 \ + --hash=sha256:ae28183297fb0d2b8dc46a1f01d51f5e45825fc5afe76a835a6cb7fb34821295 \ + --hash=sha256:b0135bcdcaa0f23573f286e4cb5e0fd2962700964ed13df085b85f2b97aeab9e \ + --hash=sha256:b087b1c5be7ac687cf22eabfe4b6b608d40df23610651e93611e1f49118baf84 \ + --hash=sha256:b0d955195bbbe489ad343fcc956eacea9357b79cb22192c66cacdefcbc14b32f \ + --hash=sha256:b281a3b0f0822618fe5e3e0d8a2048b6356b14388505dc9374ccffeb69989713 \ + --hash=sha256:b6d0c2183008c188e19f4906d426b293bdc4f67ab17df8e180fe16cda208fa71 \ + --hash=sha256:bbce99252ba3167b2b6277f1829d5bf4b43b754524bddf7f944707c3db7d2253 \ + --hash=sha256:bc1f08f68dac9f9e83845a8039880aba2ab553eb9b2259c3243a313182c253fe \ + --hash=sha256:bc94f474417604bd383d2cd445d071b07dd55fedceed3ce33407bf1fcc107290 \ + --hash=sha256:bed5163e03b98bc1fa2eb05d74c63d9c5c95d8ed6254985481640fbf5e237dea \ + --hash=sha256:c17799a62c142d61b8a3c51752a7cbc87fe2ad4ccfab10e628a77b405075c662 \ + --hash=sha256:c18db21573bd2c6489f9a544b7499f0df2853958c568e5e783536ee1f690af41 \ + --hash=sha256:c3ede305158e75510be50869b319550ab072008c13d64d4ab1e094fb286b6f44 \ + --hash=sha256:c516cc5367ca3448995d42cb994bf3f4c9002d2a7c22eac9622551269ad1b807 \ + --hash=sha256:c531166c42ea7bdfecc8c50049581f05dd1993b09cc7c52bb36a14e96deaec7d \ + --hash=sha256:c73622ef819328873b53109ee4f77ceb598bffedd02daf916102be3228866b78 \ + --hash=sha256:c8dce1f1e0e5358b682a6ad3fa5e31b31d4560997b8e61417e9217c8d60f8a0c \ + --hash=sha256:cb57f304525a5e3c13333b772bf9a473f36326e9c821b2e8e1b2fd36f80ae2c3 \ + --hash=sha256:ce8c25ca38cc0e3d7753ba180808de2c0c8cb24eae0df64491e40921454e9831 \ + --hash=sha256:ceff0acc940be2715bd6ad17b24c0e5304abf44f6efd0f81ee8499e640f9dc86 \ + --hash=sha256:cf356f70551d40374eaffb1aa63f1eb6d2006681cbd7a9faea173ce0f4dd7cd2 \ + --hash=sha256:d2d82aa62521c55ddfb000ae70f88cdd8de974078f6024e821dfe5addd0c818f \ + --hash=sha256:d32f3acc081cc3923386d88f422cde8892335e95f034e0104bb4cf9310d9915f \ + --hash=sha256:d4193206b6587047437f6f11d7e776df23e1c1e23af2a54d9347275614791e10 \ + --hash=sha256:d5c0e32fdbce7f1e8ef4d11f655694bf5f4175c757a9f1dc2be09b8864e5bcf5 \ + --hash=sha256:d5e062c01286d861fd6a1c4ff6e063547b3e713067f2df033c0ff97ac2ca006b \ + --hash=sha256:d8f9e8a6c4ab04b78d61f78627370d834eb004b2869dcb28cfffa647b4ea1980 \ + --hash=sha256:d939de9c82e2126f7f48a7e658f8a85ed46d57662d53f44c49b8895fe94a3eb7 \ + --hash=sha256:de531ce1e2a3364e8767878b58f4ff728a434b4fde089781fe30b1e08e2396e0 \ + --hash=sha256:e50d7b94baac6c7d09927fa5ca5800a0c7ee5015c7fcff65beb3a1931b5a6e09 \ + --hash=sha256:e5faeaee74a57d32b3ab3aebad2e348f06d3ba946fc5d28c1728455f00a3d13a \ + --hash=sha256:e6f0cc1bb9900dc558960894adeb30b0c083366fc1d69b856209fb2ca5c36fe5 \ + --hash=sha256:e8e1d6ce820aa23317e8209a86bd65a540973c12dc7552b48a4f6c8e9926815e \ + --hash=sha256:ed1e728b39a383c81035b2459cfcb35d99dfb01f7d6ebe3a913bc1cc5b81e459 \ + --hash=sha256:ee6db2fbed51a7991302e8fac498cd67e336246026d0dfa84cf5166ce1412760 \ + --hash=sha256:efbecf43d321f7b9281441f1f213f7c21c66988b0e06c2730ba13ed47a46bb08 \ + --hash=sha256:f2c634642694e6a0dad2ab1d375589fa671fd442edd5caf7d9737b8f6ca22906 \ + --hash=sha256:f3377c8c2b3ce898423c5e5dd94c7982e30aa7717a7e6ab2470b9de364963709 \ + --hash=sha256:f5187624823423e1d1b82b1072ac41dc837389e18d3d0572cc19bbee46cd550a \ + --hash=sha256:f77ac30b19221cd9bd3fcfa3d4614eff93140d0572ab730cded17b64adca05f3 \ + --hash=sha256:fe90228920fd8ff2be62622b6bb8a2b11acd65046d50c6b130614b5879605a20 + # via pydantic +pygments==2.21.0 \ + --hash=sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9 \ + --hash=sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c + # via + # rich + # sphinx +pyjwt==2.15.1 \ + --hash=sha256:42d59d631f7768a1028a64c7ff581a9bf7519804daf91fc5b6c56e30eec5e193 \ + --hash=sha256:4f259e80cdfb6b3fc18a7de51fd1ef9ec79652f25019bae68975ca2468a34df8 + # via sigstore +pyopenssl==26.4.0 \ + --hash=sha256:28dfcce0162b9211413e26dfbfdf1d24317fbeba18fc93c12400a1856b2a0bc7 \ + --hash=sha256:f0eb0cb2d581d3ad2b9c489468485e7f2ab6727d08401bcf9d824c3caddf3c1c + # via sigstore +pyparsing==3.3.3 \ + --hash=sha256:928ae7e20211f3b6f3915a72f06a0cfd29ab9d24279dd6346b6b1a7146397d36 \ + --hash=sha256:ece8c00a69cf01b45d0b1dedabb469c90d8caf996d4fda40f147627a122849a4 + # via rdflib +pypi-attestations==0.0.30 \ + --hash=sha256:14ff13c76bbef02a483c2b77532da02bfd746b5141b881f355e06b9807434423 \ + --hash=sha256:b3a9c53f6cb89e5e7b5b70e6cfca97cfc66008c1ed54087355e06e40071cef21 + # via virtualenv (pyproject.toml:release) +pyproject-api==1.11.2 \ + --hash=sha256:69040bb421a3d502217bf9d5f92a94e65e1db3fe25b641d6064b2b5b9880428b \ + --hash=sha256:7bff8690a101f5f0bac3221475d214018f196ae5eea3af9c9879f4d682a3fd60 + # via tox +python-dateutil==2.9.0.post0 \ + --hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 \ + --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 + # via arrow +python-discovery==1.6.1 \ + --hash=sha256:cf87d3627dfb4412437fdd5b13eae402607722998d21567993aedbc59b23c15e \ + --hash=sha256:d43fcdef879fe795352bd13ccf8d185ba5a9f86f36cfcd00529f596e737442b3 + # via + # tox + # virtualenv +pytokens==0.4.1 \ + --hash=sha256:0fc71786e629cef478cbf29d7ea1923299181d0699dbe7c3c0f4a583811d9fc1 \ + --hash=sha256:11edda0942da80ff58c4408407616a310adecae1ddd22eef8c692fe266fa5009 \ + --hash=sha256:140709331e846b728475786df8aeb27d24f48cbcf7bcd449f8de75cae7a45083 \ + --hash=sha256:24afde1f53d95348b5a0eb19488661147285ca4dd7ed752bbc3e1c6242a304d1 \ + --hash=sha256:26cef14744a8385f35d0e095dc8b3a7583f6c953c2e3d269c7f82484bf5ad2de \ + --hash=sha256:27b83ad28825978742beef057bfe406ad6ed524b2d28c252c5de7b4a6dd48fa2 \ + --hash=sha256:292052fe80923aae2260c073f822ceba21f3872ced9a68bb7953b348e561179a \ + --hash=sha256:29d1d8fb1030af4d231789959f21821ab6325e463f0503a61d204343c9b355d1 \ + --hash=sha256:2a44ed93ea23415c54f3face3b65ef2b844d96aeb3455b8a69b3df6beab6acc5 \ + --hash=sha256:30f51edd9bb7f85c748979384165601d028b84f7bd13fe14d3e065304093916a \ + --hash=sha256:34bcc734bd2f2d5fe3b34e7b3c0116bfb2397f2d9666139988e7a3eb5f7400e3 \ + --hash=sha256:3ad72b851e781478366288743198101e5eb34a414f1d5627cdd585ca3b25f1db \ + --hash=sha256:3f901fe783e06e48e8cbdc82d631fca8f118333798193e026a50ce1b3757ea68 \ + --hash=sha256:42f144f3aafa5d92bad964d471a581651e28b24434d184871bd02e3a0d956037 \ + --hash=sha256:4a14d5f5fc78ce85e426aa159489e2d5961acf0e47575e08f35584009178e321 \ + --hash=sha256:4a58d057208cb9075c144950d789511220b07636dd2e4708d5645d24de666bdc \ + --hash=sha256:4e691d7f5186bd2842c14813f79f8884bb03f5995f0575272009982c5ac6c0f7 \ + --hash=sha256:5502408cab1cb18e128570f8d598981c68a50d0cbd7c61312a90507cd3a1276f \ + --hash=sha256:584c80c24b078eec1e227079d56dc22ff755e0ba8654d8383b2c549107528918 \ + --hash=sha256:5ad948d085ed6c16413eb5fec6b3e02fa00dc29a2534f088d3302c47eb59adf9 \ + --hash=sha256:670d286910b531c7b7e3c0b453fd8156f250adb140146d234a82219459b9640c \ + --hash=sha256:682fa37ff4d8e95f7df6fe6fe6a431e8ed8e788023c6bcc0f0880a12eab80ad1 \ + --hash=sha256:6d6c4268598f762bc8e91f5dbf2ab2f61f7b95bdc07953b602db879b3c8c18e1 \ + --hash=sha256:79fc6b8699564e1f9b521582c35435f1bd32dd06822322ec44afdeba666d8cb3 \ + --hash=sha256:8bdb9d0ce90cbf99c525e75a2fa415144fd570a1ba987380190e8b786bc6ef9b \ + --hash=sha256:8fcb9ba3709ff77e77f1c7022ff11d13553f3c30299a9fe246a166903e9091eb \ + --hash=sha256:941d4343bf27b605e9213b26bfa1c4bf197c9c599a9627eb7305b0defcfe40c1 \ + --hash=sha256:967cf6e3fd4adf7de8fc73cd3043754ae79c36475c1c11d514fc72cf5490094a \ + --hash=sha256:970b08dd6b86058b6dc07efe9e98414f5102974716232d10f32ff39701e841c4 \ + --hash=sha256:97f50fd18543be72da51dd505e2ed20d2228c74e0464e4262e4899797803d7fa \ + --hash=sha256:9bd7d7f544d362576be74f9d5901a22f317efc20046efe2034dced238cbbfe78 \ + --hash=sha256:add8bf86b71a5d9fb5b89f023a80b791e04fba57960aa790cc6125f7f1d39dfe \ + --hash=sha256:b35d7e5ad269804f6697727702da3c517bb8a5228afa450ab0fa787732055fc9 \ + --hash=sha256:b49750419d300e2b5a3813cf229d4e5a4c728dae470bcc89867a9ad6f25a722d \ + --hash=sha256:d31b97b3de0f61571a124a00ffe9a81fb9939146c122c11060725bd5aea79975 \ + --hash=sha256:d70e77c55ae8380c91c0c18dea05951482e263982911fc7410b1ffd1dadd3440 \ + --hash=sha256:d9907d61f15bf7261d7e775bd5d7ee4d2930e04424bab1972591918497623a16 \ + --hash=sha256:da5baeaf7116dced9c6bb76dc31ba04a2dc3695f3d9f74741d7910122b456edc \ + --hash=sha256:dc74c035f9bfca0255c1af77ddd2d6ae8419012805453e4b0e7513e17904545d \ + --hash=sha256:dcafc12c30dbaf1e2af0490978352e0c4041a7cde31f4f81435c2a5e8b9cabb6 \ + --hash=sha256:ee44d0f85b803321710f9239f335aafe16553b39106384cef8e6de40cb4ef2f6 \ + --hash=sha256:f66a6bbe741bd431f6d741e617e0f39ec7257ca1f89089593479347cc4d13324 + # via black +pyyaml==6.0.3 \ + --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ + --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \ + --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ + --hash=sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956 \ + --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ + --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \ + --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ + --hash=sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a \ + --hash=sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0 \ + --hash=sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b \ + --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ + --hash=sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 \ + --hash=sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7 \ + --hash=sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e \ + --hash=sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007 \ + --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ + --hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 \ + --hash=sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9 \ + --hash=sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295 \ + --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ + --hash=sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0 \ + --hash=sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e \ + --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ + --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ + --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ + --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ + --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ + --hash=sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b \ + --hash=sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69 \ + --hash=sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5 \ + --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \ + --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \ + --hash=sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369 \ + --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \ + --hash=sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 \ + --hash=sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198 \ + --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ + --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ + --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ + --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ + --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \ + --hash=sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b \ + --hash=sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00 \ + --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ + --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ + --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \ + --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ + --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ + --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \ + --hash=sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4 \ + --hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b \ + --hash=sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf \ + --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ + --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ + --hash=sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8 \ + --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ + --hash=sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da \ + --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ + --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ + --hash=sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c \ + --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ + --hash=sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f \ + --hash=sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917 \ + --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ + --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ + --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ + --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ + --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ + --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ + --hash=sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3 \ + --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \ + --hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 \ + --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 + # via + # libcst + # spdx-tools +rdflib==7.6.0 \ + --hash=sha256:30c0a3ebf4c0e09215f066be7246794b6492e054e782d7ac2a34c9f70a15e0dd \ + --hash=sha256:6c831288d5e4a5a7ece85d0ccde9877d512a3d0f02d7c06455d00d6d0ea379df + # via spdx-tools +referencing==0.37.0 \ + --hash=sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231 \ + --hash=sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8 + # via + # cyclonedx-python-lib + # jsonschema + # jsonschema-specifications +requests==2.34.2 \ + --hash=sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0 \ + --hash=sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed + # via + # pypi-attestations + # sigstore + # sphinx +rfc3161-client==1.0.9 \ + --hash=sha256:03be7ac7919c7f4c566076b07773b81b1f1b28d447fa91c6960fb5f82fac5bc1 \ + --hash=sha256:0ac247d5aa22f3514d1e284bdc547d2ad5f101e1e5d7f6db25db2ab3a0c59f06 \ + --hash=sha256:4d33f5aaec2974239e1c62a79a6fcf32506b09095482377a8b5be3a00fa67dfb \ + --hash=sha256:5e619d7e28ffa3a46685923e6d6ba4ce2e74cb09a41c77995a0702cc1a533bc4 \ + --hash=sha256:614da494c6f8850c5d6d98c23bc291a4e4dbf2c3c9082d9907fa156f9e4a75c6 \ + --hash=sha256:66997415086d2c6e5d57a2d315157b32a9c3f4f1312877ca8eb55e2e0ddf8ce4 \ + --hash=sha256:7b703b233dbee7228d117d5f1c118363c61d523fa6f018c9217c79e3c6684f86 \ + --hash=sha256:8208999e94e1252e3f5f748f5efe59d3b9c308d04c7626be1becfbf3236c9de4 \ + --hash=sha256:908f6a775da4bdce1d39f825a4431f389542696078ef3e4ed911a068bcaf1792 \ + --hash=sha256:a7004edfbf1bb7ec7801978a39fa7a7fe385ca59673df90d7578079b6eba3b24 \ + --hash=sha256:d3cbf1991b6f3457efcb25e1d31179cac556112e6d3e65ec9e0fd70c6e9af90d \ + --hash=sha256:d4cf92f0e784ff02ff5ea1c574f8521f30953dc178092424126dbd8348034f93 \ + --hash=sha256:fac3f440a507555e684dc5daba75e33dc08f0f45fdefa48448f19001233a6b21 + # via sigstore +rfc3339-validator==0.1.4 \ + --hash=sha256:138a2abdf93304ad60530167e51d2dfb9549521a836871b88d7f4695d0022f6b \ + --hash=sha256:24f6ec1eda14ef823da9e36ec7113124b39c04d50a4d3d3a3c2859577e7791fa + # via jsonschema +rfc3986==2.0.0 \ + --hash=sha256:50b1502b60e289cb37883f3dfd34532b8873c7de9f49bb546641ce9cbd256ebd \ + --hash=sha256:97aacf9dbd4bfd829baad6e6309fa6573aaf1be3f6fa735c8ab05e46cecb261c + # via pypi-attestations +rfc3986-validator==0.1.1 \ + --hash=sha256:2f235c432ef459970b4306369336b9d5dbdda31b510ca1e327636e01f528bfa9 \ + --hash=sha256:3d44bde7921b3b9ec3ae4e3adca370438eccebc676456449b145d533b240d055 + # via jsonschema +rfc3987-syntax==1.1.0 \ + --hash=sha256:6c3d97604e4c5ce9f714898e05401a0445a641cfa276432b0a648c80856f6a3f \ + --hash=sha256:717a62cbf33cffdd16dfa3a497d81ce48a660ea691b1ddd7be710c22f00b4a0d + # via jsonschema +rfc8785==0.1.4 \ + --hash=sha256:520d690b448ecf0703691c76e1a34a24ddcd4fc5bc41d589cb7c58ec651bcd48 \ + --hash=sha256:e545841329fe0eee4f6a3b44e7034343100c12b4ec566dc06ca9735681deb4da + # via sigstore +rich==15.0.0 \ + --hash=sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb \ + --hash=sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36 + # via sigstore +roman==5.2 \ + --hash=sha256:275fe9f46290f7d0ffaea1c33251b92b8e463ace23660508ceef522e7587cb6f \ + --hash=sha256:89d3b47400388806d06ff77ea77c79ab080bc127820dea6bf34e1f1c1b8e676e + # via docstrfmt +roman-numerals==4.1.0 \ + --hash=sha256:1af8b147eb1405d5839e78aeb93131690495fe9da5c91856cb33ad55a7f1e5b2 \ + --hash=sha256:647ba99caddc2cc1e55a51e4360689115551bf4476d90e8162cf8c345fe233c7 + # via sphinx +rpds-py==2026.6.3 \ + --hash=sha256:0be972be84cfcaf46c8c6edf690ca0f154ac17babf1f6a955a51579b34ad2dc5 \ + --hash=sha256:127565fead0a10943b282957bd5447804ff3160ad79f2ad2635e6d249e380680 \ + --hash=sha256:127e08c0642d880cf32ca47ec2a4a77b901f7e2dd1ad9762adb13955d72ffcc9 \ + --hash=sha256:166cf54d9f44fc6ceb53c7860258dde44a81406646de79f8ed3234fca3b6e538 \ + --hash=sha256:168c733a7112e071bb7a66460e667edfcff06c017a3c523f7a8a8e08d0140804 \ + --hash=sha256:1967debc37f64f2c4dc90a7f563aec558b471966e12adcac4e1c4240496b6ebf \ + --hash=sha256:1cebd1337c242e4ec2293e541f712b2da849b29f48f0c293684b71c0632625d4 \ + --hash=sha256:1cf01971c4f2c5553b772a542e4aaf191789cd331bc2cd4ff0e6e65ba49e1e97 \ + --hash=sha256:1e5822dfc2f0d4ab7e745eaa6d85945069329beeccef965af3f3bb26058fcab6 \ + --hash=sha256:22bffe6042b9bcb0822bcd1955ec00e245daf17b4344e4ed8e9551b976b63e96 \ + --hash=sha256:23a439f31ccbeff1574e24889128821d1f7917470e830cf6544dced1c662262a \ + --hash=sha256:24e9c5386e16669b674a69c156c8eeefcb578f3b3397b713b08e6d60f3c7b187 \ + --hash=sha256:270b293dae9058fc9fcedab50f13cebf46fb8ed1d1d54e0521a9da5d6b211975 \ + --hash=sha256:29dfa0533a5d4c94d4dfa1b694fcb56c9c63aad8330ffdd816fd225d0a7a162f \ + --hash=sha256:2a9c6f195058cb45335e8cc3802745c603d716eb96bc9625950c1aac71c0c703 \ + --hash=sha256:2bfd04c19ddbd6640de0b51894d764bd2758854d5b75bd102d2ef10cb9c293a9 \ + --hash=sha256:2c54a076ca4d370980ab57bc0e31df57bbe8d41340436a90ef8b1219a3cbb127 \ + --hash=sha256:2c958bf94822e9290a40aaf2a822d4bc5c88099093e3948ad6c571eca9272e5f \ + --hash=sha256:2c99f7e8ccb3dd6e3e4bfeac657a7b208c9bac8075f4b078c02d7404c34107fa \ + --hash=sha256:2f7c26fbc5acd2522b95d4177fe4710ffd8e9b20529e703ffbf8db4d93903f05 \ + --hash=sha256:30c6dc199b24a5e3e81d50da0f00858c5bbdb2617a750395687f4339c5818171 \ + --hash=sha256:38a2fea2787428f811719ceb9114cb78964a3138838320c29ac39526c79c16ba \ + --hash=sha256:3a83ae6c67b7676b9878378547ca8e93ed77a580037bcbcd1d32f739e1e6089c \ + --hash=sha256:3cfe765c1da0072636ca06628261e0ea05688e160d5c8a03e0217c3854037223 \ + --hash=sha256:421aba32367055614287a4292b6a17f1939c9452299f7a0209c117e990b646d4 \ + --hash=sha256:425560c6fa0415f27261727bb20bd097568485e5eb0c121f1949417d1c516885 \ + --hash=sha256:4470ce197d4090875cf6affbf1f853338387428df97c4fb7b7106317b8214698 \ + --hash=sha256:4cf2d36a2357e4d07bb5a4f98801265327b48256867816cfd2ceb001e9754a8f \ + --hash=sha256:4f4bca01b63096f606e095734dd56e74e175f94cfbf24ff3d63281cec61f7bb7 \ + --hash=sha256:501f9f04a588d6a09179368c57071301445191767c64e4b52a6aa9871f1ef5ed \ + --hash=sha256:536bceea4fa4acf7e1c61da2b5786304367c816c8895be71b8f537c480b0ea1f \ + --hash=sha256:538949e262e46caa31ac01bdb3c1e8f642622922cacbabbae6a8445d9dc33eaf \ + --hash=sha256:539d75de9e0d536c84ff18dfeb805398e58227001ce09231a26a08b9aed1ee0e \ + --hash=sha256:54f45a148e28767bf343d33a684693c70e451c6f4c0e9904709a723fafbdfc1f \ + --hash=sha256:55927d532399c2c646100ff7feb48eaa940ad70f42cd68e1328f3ded9f81ca24 \ + --hash=sha256:58eadac9cd119677b60e1cf8ac4052f35949d71b8a9e5556efccbe82533cf22a \ + --hash=sha256:5e8d07bddee435a2ff6f1920e18feff28d0bc4533e42f4bf6927fbd073312c41 \ + --hash=sha256:62698275682bf121181861295c9181e789030a2d516071f5b8f3c23c170cd0fc \ + --hash=sha256:639c8929aa0afe81be836b04de888460d6bed38b9c54cfc18da8f6bfabf5af5d \ + --hash=sha256:67e3a721ffc5d8d2210d3671872298c4a84e4b8035cfe42ffd7cde35d772b146 \ + --hash=sha256:6de4744d05bd1aa1be4ed7ea1189e3979196808008113bbbf899a460966b925e \ + --hash=sha256:6e84adbcf4bf841aed8116a8264b9f50b4cb3e7bd89b516122e616ac56ca269e \ + --hash=sha256:7491ee23305ac3eb59e492b6945881f5cd77a6f731061a3f25b77fd40f9e99a4 \ + --hash=sha256:79486287de1730dbaff3dbd124d0ca4d2ef7f9d29bf2544f1f93c09b5bcbbd12 \ + --hash=sha256:7b689145a1485c335569bd056464f3243a29af7ed3871c7be31ad624ba239bc7 \ + --hash=sha256:7f88d653e7b3b779d71ae7454e20dcc9b6bae903f33c269db9f2be41bda3f261 \ + --hash=sha256:8020133a74bd81b4572dd8e4be028a6b1ebcd70e6726edc3918008c08bee6ee6 \ + --hash=sha256:808345f53cb952433ca2816f1604ff3515608a81784954f38d4452acfe8e61d5 \ + --hash=sha256:83e35b57523816c8613fd0776b40cd8bb9f596b37ddd2692eb4a6bb5ab2f8c93 \ + --hash=sha256:842e7b070435622248c7a2c44ae53fa1440e073cc3023bc919fed570884097a7 \ + --hash=sha256:847927daf4cffbd4e90e42bc890069897101edd015f956cb8721b3473372edda \ + --hash=sha256:882076c00c0a608b131187055ddc5ae29f2e7eaf870d6168980420d58528a5c8 \ + --hash=sha256:8b95977e7211527ab0ba576e286d023389fbeeb32a6b7b771665d333c60e5342 \ + --hash=sha256:8bb68f03f395eb793220b45c097bd4d8c32944393da0fad8b999efac0868fc8c \ + --hash=sha256:8c2642a7603ec0b16ed77da4555db3b4b472341904873788327c0b0d7b95f1bb \ + --hash=sha256:8c3d1e9c15b9d51ca0391e13da1a25a0a4df3c58a37c9dc368e0736cf7f69df0 \ + --hash=sha256:8c6e5a2f750cc71c3e3b11d71661f21d6f9bc6cebc6564b1466417a1ec03ec77 \ + --hash=sha256:8d2294a31386bfa251d8c8a39472beee17db67d4f1a6eabea665d35c9a4461c3 \ + --hash=sha256:8e4320744c1ffdd95a603def63344bfab2d33edeab301c5007e7de9f9f5b3885 \ + --hash=sha256:8e65860d238379ed982fd9ba690579b5e95af2f4840f99c772816dbe573cb826 \ + --hash=sha256:8f2e5c5ee828d42cb11760761c0af6507927bec42d0ad5458f97c9203b054617 \ + --hash=sha256:900a67df3fd1660b035a4761c4ce73c382ea6b35f90f9863c36c6fd8bf8b09bb \ + --hash=sha256:913ca42ccad3f8cc6e292b587ae8ae49c8c823e5dce51a736252fc7c7cdfa577 \ + --hash=sha256:9250a9a0a6fd4648b3f868da8d91a4c52b5811a62df58e753d50ae4454a36f80 \ + --hash=sha256:931908d9fc855d8f74783377822be318edb6dcb19e47169dc038f9a1bf60b06e \ + --hash=sha256:9826217f048f620d9a712672818bf231442c1b35d96b227a07eabd11b4bb6945 \ + --hash=sha256:9891e594296ab9dada6551c8e7b387b2721f27a67eecd528412e8906247a7b90 \ + --hash=sha256:9c1255b302953c86a486b81d330d5ee1d5bd937691ce271b6be0ef0e299eaab7 \ + --hash=sha256:a0811d33247c3d6128a3001d763f2aa056bb3425204335400ac54f89eec3a0d0 \ + --hash=sha256:a136d453475ac0fcbda502ef1e6504bd28d6d904700915d278deeab0d00fe140 \ + --hash=sha256:a214c993455f99a89aaeadc9b21241900037adc9d97203e374d75513c5911822 \ + --hash=sha256:a3086b538543802f84c843911242db20447de00d8752dd0efc936dbcf02218ba \ + --hash=sha256:a3450b693fde92133e9f51060568a4c31fcca76d5e53bbd611e689ca446517e9 \ + --hash=sha256:a550fb4950a06dde3beb4721f5ad4b25bf4513784665b0a8522c792e2bd822a4 \ + --hash=sha256:a9f4645593036b81bbdb36b9c8e0ea0d1c3fee968c4d59db0344c14087ef143a \ + --hash=sha256:aca6c1ef08a82bfe327cc156da694660f599923e2e6665b6d81c9c2d0ac9ffc8 \ + --hash=sha256:acac386b453c2516111b50985d60ce46e7fadb5ea71ae7b25f4c946935bf27cf \ + --hash=sha256:acc992ab27b15f852c76755eb2ab7dce86585ddadba6fa5946e58556088845b4 \ + --hash=sha256:ae3d4fe8c0b9213624fdce7279d70e3b148b682ca20719ebd193a23ebfa47324 \ + --hash=sha256:ae50181a047c871561212bb97f7932a2d45fb53e947bd9b57ebad85b529cbc53 \ + --hash=sha256:ae6dd8f10bd17aad820876d24caec9efdafd80a318d16c0a48edb5e136902c6b \ + --hash=sha256:af05d726809bff6b141be124d4c7ce998f9c9c7f30edb1f46c07aa103d540b41 \ + --hash=sha256:afd70d95892096cdb26f15a00c45907b17817577aa8d1c76b2dcc2788391f9e9 \ + --hash=sha256:b5c2dc92304aa48a4a60443b548bb12f12e119d4b72f314015e67b9e1be97fca \ + --hash=sha256:bc0011654b91cc4fb2ae701bec0a0ba1e552c0714247fa7af6c59e0ccfa3a4e1 \ + --hash=sha256:bcfbcf66006befb9fd2aeaa9e01feaf881b4dc330a02ba07d2322b1c11be7b5d \ + --hash=sha256:bdbd97738551fca3917c1bd7188bec1920bb520104f28e7e1007f9ceb17b7690 \ + --hash=sha256:c60924535c75f1566b6eb75b5c31a48a43fef04fa2d0d201acbad8a9969c6107 \ + --hash=sha256:c7b9a2f8f4d8e90af72571d3d495deebdd7e3c75451f5b41719aee166e940fc2 \ + --hash=sha256:ca6546b66be9dc4738b1b043d5ebd5488c66c578c5ff0fd0e8065313fe3afb76 \ + --hash=sha256:ccffae9a092a00deb7efd545fe5e2c33c33b88e7c054337e9a74c179347d0b7d \ + --hash=sha256:cdc7e35386f3847df728fbcb5e887e2d79c19e2fa1eba9e51b6621d23e3243af \ + --hash=sha256:d15fde0e6fb0d88a60d221204873743e5d9f0b7d29165e62cd86d0413ad74ba6 \ + --hash=sha256:d34c20167764fbcf927194d532dd7e0c56772f0a5f943fa5ef9e9afbba8fb9db \ + --hash=sha256:d483fe17f01ad64b7bf7cc38fcefff1ca9fb83f8c2b2542b68f97ffe0611b369 \ + --hash=sha256:d7469697dce35be237db177d42e2a2ee26e6dcc5fc052078a6fefabd288c6edd \ + --hash=sha256:db08f45aecde626498fb3df07bcf6d2ec040af42e859a4f5040d79c200342911 \ + --hash=sha256:dc319e5a1de4b6913aac94bf6a2f9e847371e0a140a43dd4991db1a09bc2d504 \ + --hash=sha256:de3eceba0b683bcbb1ab93da016d0270df1f9ae7be716b40214c5dafac6ea45a \ + --hash=sha256:dfcc8b909769d19db55c7cc9541eb64b9b774b1057ffffb4f1048070475bb9f9 \ + --hash=sha256:e059c5dde6452b44424bd1834557556c226b57781dee1227af23518459722b13 \ + --hash=sha256:e4316bf32babbed84e691e352faf967ce2f0f024174a8643c37c94a1080374fc \ + --hash=sha256:e52655eaf81e32593abedaa4bfe33170c8cfedf3365ed9be6e11e07f148f0278 \ + --hash=sha256:e55d236be29255554da47abe5c577637db7c24a02b8b46f0ca9524c855801868 \ + --hash=sha256:ea7bb13b7c9a29791f87a0387ba7d3ad3a6d783d827e4d3f27b40a0ff44495e2 \ + --hash=sha256:ea964164cc9afa72d4d9b23cc28dafae93693c0a53e0b42acbff15b22c3f9ddd \ + --hash=sha256:ec829541c45bca16e61c7ae50c20501f213605beb75d1aba91a6ee37fbbb56a4 \ + --hash=sha256:ecabd69db66de867690f9797f2f8fa27ba501bbc24540cbdbdc649cd15888ba6 \ + --hash=sha256:ed0c1e5d10cdc7135537988c74a0188da68e2f3c30813ba3744ab1e42e0480f9 \ + --hash=sha256:f0840b5b17057f7fd918b76183a4b5a0635f43e14eb2ce60dce1d4ee4707ea00 \ + --hash=sha256:f4d78253f6996be4901669ad25319f842f740eccf4d58e3c7f3dd39e6dde1d8f \ + --hash=sha256:f56f1695bc5c0871cbc33dc0130fcf503aab0c57dcc5a6700a4f49eba4f2652e \ + --hash=sha256:f826877d462181e5eb1c26a0026b8d0cab05d99844ecb6d8bf3627a2ca0c0442 \ + --hash=sha256:f8f23ead891a3b762f35ab3b04623da7056545b48aa60d59957e6789914545da \ + --hash=sha256:f90938e92afda60266da758ee7d363447f7f0138c9559f9e1811629580582d90 \ + --hash=sha256:faa679d19a6696fd54259ad321251ad77a13e70e03dd834daa762a44fb6196ef + # via + # jsonschema + # referencing +securesystemslib==1.5.1 \ + --hash=sha256:4b8d00abd93707ead10b69eb2b8582376a1364de3b0a71077de534c2ef4985e0 \ + --hash=sha256:ada8bdf817da29ece4ba91654f6a162ce7cfbadbc3ae3f840f7313f9d22675de + # via tuf +semantic-version==2.10.0 \ + --hash=sha256:bdabb6d336998cbb378d4b9db3a4b56a1e3235701dc05ea2690d9a997ed5041c \ + --hash=sha256:de78a3b8e0feda74cabc54aab2da702113e33ac9d9eb9d2389bcf1f58b7d9177 + # via spdx-tools +setuptools==84.0.0 \ + --hash=sha256:51a52592b3b99e102b609654876bd65f19f999935166d1352678931132b0c670 \ + --hash=sha256:f4695c21257f0d9b537ec2692c941d02ee143b7cc1276941349a546573b2ef73 + # via setuptools-scm +setuptools-scm==10.3.4 \ + --hash=sha256:82f34c3e3084fc2b57d397200637cc13f2338004052d2fee47baa5f0e902464e \ + --hash=sha256:a69f28bfc245608781205e912faae437c2b2165773afa4e7b979d77447a69dd2 + # via hatch-vcs +sigstore==4.5.0 \ + --hash=sha256:020d3e07f622b2916bf453e66ff6ff0711e1fdc5ab69e8bd8902f71d9fcb316f \ + --hash=sha256:f045b207f2e12605cf775ec38e89c5eda625d71ffa7830477db65e47ec2bc8b2 + # via pypi-attestations +sigstore-models==0.0.6 \ + --hash=sha256:5201a68f4d7d0f8bec1e2f4378eb646b084c52609a4e31db8c385095fff68b2e \ + --hash=sha256:c766c09470c2a7e8a4a333c893f07e2001c56a3ff1757b1a246119f53169a849 + # via + # pypi-attestations + # sigstore +sigstore-rekor-types==0.0.18 \ + --hash=sha256:19aef25433218ebf9975a1e8b523cc84aaf3cd395ad39a30523b083ea7917ec5 \ + --hash=sha256:b62bf38c5b1a62bc0d7fe0ee51a0709e49311d137c7880c329882a8f4b2d1d78 + # via sigstore +six==1.17.0 \ + --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ + --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 + # via + # python-dateutil + # rfc3339-validator +smmap==5.0.3 \ + --hash=sha256:4d9debb8b99007ae47165abc08670bd74cb74b5227dda7f643eccc4e9eb5642c \ + --hash=sha256:c106e05d5a61449cf6ba9a1e650227ecfb141590d2a98412103ff35d89fc7b2f + # via gitdb +snowballstemmer==3.1.1 \ + --hash=sha256:7e207fa178741da09cdee59d3ecec3827ad5f92b1fc5c9ff3755b639f71f5752 \ + --hash=sha256:e07bbc54a0d798fe6010a12398422e62a8bfbba95c394fd0956ef58cb4d3e260 + # via sphinx +sortedcontainers==2.4.0 \ + --hash=sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88 \ + --hash=sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0 + # via cyclonedx-python-lib +spdx-tools==0.8.5 \ + --hash=sha256:7c2d5865941be9d2e898f5b084e8d5422dd298dc5a29320ddb198fec304f59c4 \ + --hash=sha256:be600beb2f762f0116025e05490d399e724f668bef84025a7c421bb266688bdb + # via virtualenv (pyproject.toml:release) +sphinx==9.1.0 \ + --hash=sha256:7741722357dd75f8190766926071fed3bdc211c74dd2d7d4df5404da95930ddb \ + --hash=sha256:c84fdd4e782504495fe4f2c0b3413d6c2bf388589bb352d439b2a3bb99991978 + # via docstrfmt +sphinxcontrib-applehelp==2.0.0 \ + --hash=sha256:2f29ef331735ce958efa4734873f084941970894c6090408b079c61b2e1c06d1 \ + --hash=sha256:4cd3f0ec4ac5dd9c17ec65e9ab272c9b867ea77425228e68ecf08d6b28ddbdb5 + # via sphinx +sphinxcontrib-devhelp==2.0.0 \ + --hash=sha256:411f5d96d445d1d73bb5d52133377b4248ec79db5c793ce7dbe59e074b4dd1ad \ + --hash=sha256:aefb8b83854e4b0998877524d1029fd3e6879210422ee3780459e28a1f03a8a2 + # via sphinx +sphinxcontrib-htmlhelp==2.1.0 \ + --hash=sha256:166759820b47002d22914d64a075ce08f4c46818e17cfc9470a9786b759b19f8 \ + --hash=sha256:c9e2916ace8aad64cc13a0d233ee22317f2b9025b9cf3295249fa985cc7082e9 + # via sphinx +sphinxcontrib-jsmath==1.0.1 \ + --hash=sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178 \ + --hash=sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8 + # via sphinx +sphinxcontrib-qthelp==2.0.0 \ + --hash=sha256:4fe7d0ac8fc171045be623aba3e2a8f613f8682731f9153bb2e40ece16b9bbab \ + --hash=sha256:b18a828cdba941ccd6ee8445dbe72ffa3ef8cbe7505d8cd1fa0d42d3f2d5f3eb + # via sphinx +sphinxcontrib-serializinghtml==2.0.0 \ + --hash=sha256:6e2cb0eef194e10c27ec0023bfeb25badbbb5868244cf5bc5bdc04e4464bf331 \ + --hash=sha256:e9d912827f872c029017a53f0ef2180b327c3f7fd23c87229f7a8e8b70031d4d + # via sphinx +tabulate==0.10.0 \ + --hash=sha256:e2cfde8f79420f6deeffdeda9aaec3b6bc5abce947655d17ac662b126e48a60d \ + --hash=sha256:f0b0622e567335c8fabaaa659f1b33bcb6ddfe2e496071b743aa113f8774f2d3 + # via docstrfmt +tomli-w==1.2.0 \ + --hash=sha256:188306098d013b691fcadc011abd66727d3c414c571bb01b1a174ba8c983cf90 \ + --hash=sha256:2dd14fac5a47c27be9cd4c976af5a12d87fb1f0b4512f81d69cce3b35ae25021 + # via tox +tomlkit==0.15.1 \ + --hash=sha256:177a05aece5a8ca5266fd3c448abb47b8d352f09d477d3ca8332db4d89b24304 \ + --hash=sha256:e25bbf38843005246210a12982776f27f99cb9be67160e14434d0c0d21ee1e97 + # via hatchling +towncrier==26.9.0 \ + --hash=sha256:ace9031631c718cc0709107b8755e6e31350eb8acdc877af43df3177addb9a55 \ + --hash=sha256:ae4d223e6aadaff98d29b7f09e58d9c9ccf4cd3b455a2d2e0486d432cd8bf660 + # via virtualenv (pyproject.toml:release) +tox==4.64.7 \ + --hash=sha256:4e7a906f26b72fa9bf11577d02bde492c6d0ad8e690921a801893354d05d35fe \ + --hash=sha256:b8033a00945a02493adebdc337e4e45136e21a7083c00b2ae27c998370b72af3 + # via + # virtualenv (pyproject.toml:release) + # tox-uv-bare +tox-uv==1.36.1 \ + --hash=sha256:6996cb49d0b6b838ae015b76611770595cb6f658a27e2a9661766148194c18bd + # via virtualenv (pyproject.toml:release) +tox-uv-bare==1.36.1 \ + --hash=sha256:33a08e4205c63811dd76590590718539fe87dcadda5882f164f747bd5db31a2c \ + --hash=sha256:de41cf5450b8f37837fd68834dfebef59939cd10fde030e498cd9a48db19f1e8 + # via tox-uv +trove-classifiers==2026.9.21.13 \ + --hash=sha256:0a9ebc8d4e2f3e8a22848c5258033035bec17a3012ac3fea16dbaa764489eb71 \ + --hash=sha256:8b1ff4f9c191b1040b71c37f1e445ab99732911e3cd91de52838453a854d7a17 + # via hatchling +tuf==7.0.1 \ + --hash=sha256:5ada1db78da3518fa851e28588f4b2249b8662a28ae528f54afa2a1de541fb31 \ + --hash=sha256:d30434bda6e079ab303fb30d1b3006d939a10ca34783b1573d61cd9b802fa45c + # via sigstore +types-docutils==0.22.3.20260724 \ + --hash=sha256:0223ce87f9b8331a5a3a7c7832e828033d289da6b878a0dcfbc74c30ec58850e \ + --hash=sha256:45e2fe584608671aa648784c4f805d08a36cd69eb2bd542e60522140c46dc89c + # via docstrfmt +typing-extensions==4.16.0 \ + --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \ + --hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5 + # via + # pydantic + # pydantic-core + # sigstore-models + # typing-inspection +typing-inspection==0.4.4 \ + --hash=sha256:547274fa6b0a561ccf549cc9524b999a578e737d015d8709d021f9d0d13bea47 \ + --hash=sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147 + # via pydantic +tzdata==2026.4 \ + --hash=sha256:c2169a8b0a7a5e9674da5a135ccdfb2b3e671b333ed9fed17b41f73c34476e81 \ + --hash=sha256:f1b8bd365d8d210c55353f4d7f8d6d8561c0ba50d704b700d195a9424bba0d79 + # via arrow +uri-template==1.3.0 \ + --hash=sha256:0e00f8eb65e18c7de20d595a14336e9f337ead580c70934141624b6d1ffdacc7 \ + --hash=sha256:a44a133ea12d44a0c0f06d7d42a52d71282e77e2f937d8abd5655b8d56fc1363 + # via jsonschema +uritools==6.1.3 \ + --hash=sha256:136f113e76e53f85bf2d9cce0c3d40ceec775d0409e7d6de9b28f046a7d42838 \ + --hash=sha256:3a498e7e85ef3249343d5710618d641a414da0fbae6d23053ada7976ee83ea5f + # via spdx-tools +urllib3==2.8.0 \ + --hash=sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3 \ + --hash=sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63 + # via + # id + # requests + # tuf +uv==0.12.22 \ + --hash=sha256:0247b8795348fe16fec7d143b2e4d91a953e69644993a557bb5238b9f53b6a28 \ + --hash=sha256:02774878a9c0ddc374ff1cdb7fc2dee5cd499307223c2d2f0a730274b5794546 \ + --hash=sha256:04af20f84c122039c16a006b497ea25d4850543b3ac2bd879d96d36d9890bebc \ + --hash=sha256:0621be2adfedf7673e46a708b1c781b0e4c61e715e33b3196ad32afffcc5aef6 \ + --hash=sha256:1c45b1e544901b9eef14806ad964cd1aca49338748e3d8030e41c4aed8b17b00 \ + --hash=sha256:31096da4ffd7d532909e046cf8acb606ad8e770019404b710a58310bec073119 \ + --hash=sha256:3e1e70abbeeb85ad6589e9b8c2cee2785782a26ec220705b60ac8d3d802deb83 \ + --hash=sha256:40f6afd2a67d7ad0c58d7a05e9c0a58a30d333e924e3b9e9941af19bd044326a \ + --hash=sha256:53992ea1a64344bc887e3aec8d016488b9f185d96d881ea8e4e2a8ab0a259a39 \ + --hash=sha256:680769ef701922dd93426b73476073411941f51ac46991a6e5c5a6efbf68a713 \ + --hash=sha256:699c673c780037e19a623dd66d00c12853def679b5c8f383359012d582721870 \ + --hash=sha256:6acf7c5e4c28075c074a104953d61f48fef05e0075d9eb4180e5e6430e4eacfa \ + --hash=sha256:72516e1edef5950b556bdaf3b382f537e4249e06362378daa811a6eefc165a14 \ + --hash=sha256:82911ca7a8310181f382a8f614f00a45f0403561e2aa92aa135c1fa773b0fc44 \ + --hash=sha256:972cb556d34372f4c19f5c8bba246c92c5a8bfda3f30b2a5b8e12fddd81ba441 \ + --hash=sha256:992956578af9ec09d9c9d8967527dcf43e07b43dea8aecba74a20d37087feeb2 \ + --hash=sha256:cbfc9dab909f0e7fec63ae13e4215abd53636a5424aabc988fd4974ea124a3ad \ + --hash=sha256:daff5ceafdee5cb268b6d4e991ea150fcad5dff72b97d3e5b84d78d04fcf8482 \ + --hash=sha256:db53a974617ca520cfdbf95fbbcd9cf5ab95016d601c1ecf788295c01053b8d6 + # via + # virtualenv (pyproject.toml:release) + # tox-uv +vcs-versioning==2.5.0 \ + --hash=sha256:956a796e31f80fe714d219d6d1df15a6bf247d10f6d851bf4b98279d0a42da55 \ + --hash=sha256:dbf44f6303dc817e5792cb7d0e9cc90e3046ee1e32eaa357b20ad368811313d0 + # via setuptools-scm +virtualenv==21.14.3 \ + --hash=sha256:4acc848a57a3bdb2ae0873d0689664bccad2230784de3753f57128eb126d3934 \ + --hash=sha256:cb5cd3a87f12a6cfdc0cef6bc8bead6138221767f7718e0c00fee5fb907f9858 + # via tox +webcolors==25.10.0 \ + --hash=sha256:032c727334856fc0b968f63daa252a1ac93d33db2f5267756623c210e57a4f1d \ + --hash=sha256:62abae86504f66d0f6364c2a8520de4a0c47b80c03fc3a5f1815fedbef7c19bf + # via jsonschema +xmltodict==1.0.4 \ + --hash=sha256:6d94c9f834dd9e44514162799d344d815a3a4faec913717a9ecbfa5be1bb8e61 \ + --hash=sha256:a4a00d300b0e1c59fc2bfccb53d7b2e88c32f200df138a0dd2229f842497026a + # via spdx-tools diff --git a/tasks/release.py b/tasks/release.py index feda8b3c2..1534d34c2 100644 --- a/tasks/release.py +++ b/tasks/release.py @@ -1,51 +1,61 @@ -"""Handles creating a release PR.""" +"""Handles creating a release.""" from __future__ import annotations +import sys from pathlib import Path from subprocess import check_call -from git import Commit, Head, Remote, Repo, TagReference +from git import Commit, Remote, Repo, TagReference from packaging.version import Version ROOT_SRC_DIR = Path(__file__).resolve().parents[1] +CHANGELOG_DIR = ROOT_SRC_DIR / "docs" / "changelog" -def main(version_str: str) -> None: - version = Version(version_str) +def main(version_str: str, *, push: bool) -> None: repo = Repo(str(ROOT_SRC_DIR)) - if repo.is_dirty(): msg = "Current repository is dirty. Please commit any changes and try again." raise RuntimeError(msg) - upstream, release_branch = create_release_branch(repo, version) + remote = get_remote(repo) + remote.fetch() + version = resolve_version(version_str, repo) + print(f"releasing {version}") # ruff:ignore[print] release_commit = release_changelog(repo, version) tag = tag_release_commit(release_commit, repo, version) - print("push release commit") # noqa: T201 - repo.git.push(upstream.name, release_branch) - print("push release tag") # noqa: T201 - repo.git.push(upstream.name, tag) - print("All done! ✨ 🍰 ✨") # noqa: T201 - - -def create_release_branch(repo: Repo, version: Version) -> tuple[Remote, Head]: - print("create release branch from upstream main") # noqa: T201 - upstream = get_upstream(repo) - upstream.fetch() - branch_name = f"release-{version}" - release_branch = repo.create_head(branch_name, upstream.refs.main, force=True) - upstream.push(refspec=f"{branch_name}:{branch_name}", force=True) - release_branch.set_tracking_branch(repo.refs[f"{upstream.name}/{branch_name}"]) - release_branch.checkout() - return upstream, release_branch - - -def get_upstream(repo: Repo) -> Remote: - upstream_remote = "pypa/virtualenv.git" + if push: + print("push release commit") # ruff:ignore[print] + repo.git.push(remote.name, "HEAD:main") + print("push release tag") # ruff:ignore[print] + repo.git.push(remote.name, tag) + print("All done! ✨ 🍰 ✨") # ruff:ignore[print] + + +def resolve_version(version_str: str, repo: Repo) -> Version: + if version_str not in {"auto", "major", "minor", "patch"}: + return Version(version_str) + latest_tag = repo.git.describe("--tags", "--abbrev=0") + parts = [int(x) for x in latest_tag.split(".")] + if version_str == "major": + parts = [parts[0] + 1, 0, 0] + elif version_str == "minor": + parts = [parts[0], parts[1] + 1, 0] + elif version_str == "patch": + parts[2] += 1 + elif any(CHANGELOG_DIR.glob("*.feature.rst")) or any(CHANGELOG_DIR.glob("*.removal.rst")): + parts = [parts[0], parts[1] + 1, 0] + else: + parts[2] += 1 + return Version(".".join(str(p) for p in parts)) + + +def get_remote(repo: Repo) -> Remote: + upstream_remote = "pypa/virtualenv" urls = set() for remote in repo.remotes: for url in remote.urls: - if url.endswith(upstream_remote): + if url.rstrip(".git").endswith(upstream_remote): return remote urls.add(url) msg = f"could not find {upstream_remote} remote, has {urls}" @@ -53,18 +63,23 @@ def get_upstream(repo: Repo) -> Remote: def release_changelog(repo: Repo, version: Version) -> Commit: - print("generate release commit") # noqa: T201 - check_call(["towncrier", "build", "--yes", "--version", version.public], cwd=str(ROOT_SRC_DIR)) # noqa: S607 + print("generate release commit") # ruff:ignore[print] + check_call( + [sys.executable, "-m", "towncrier", "build", "--yes", "--version", version.public], cwd=str(ROOT_SRC_DIR) + ) + # pre-commit would resolve hook dependencies unpinned, so wrap the changelog with the locked docstrfmt + check_call([sys.executable, "-m", "docstrfmt", "-l", "120", "docs/changelog.rst"], cwd=str(ROOT_SRC_DIR)) + repo.git.add(".") return repo.index.commit(f"release {version}") -def tag_release_commit(release_commit, repo, version) -> TagReference: - print("tag release commit") # noqa: T201 +def tag_release_commit(release_commit: Commit, repo: Repo, version: Version) -> TagReference: + print("tag release commit") # ruff:ignore[print] existing_tags = [x.name for x in repo.tags] if version in existing_tags: - print(f"delete existing tag {version}") # noqa: T201 + print(f"delete existing tag {version}") # ruff:ignore[print] repo.delete_tag(version) - print(f"create tag {version}") # noqa: T201 + print(f"create tag {version}") # ruff:ignore[print] return repo.create_tag(version, ref=release_commit, force=True) @@ -72,6 +87,7 @@ def tag_release_commit(release_commit, repo, version) -> TagReference: import argparse parser = argparse.ArgumentParser(prog="release") - parser.add_argument("--version", required=True) + parser.add_argument("--version", default="auto") + parser.add_argument("--no-push", action="store_true") options = parser.parse_args() - main(options.version) + main(options.version, push=not options.no_push) diff --git a/tasks/run_graalpy_tests.py b/tasks/run_graalpy_tests.py new file mode 100644 index 000000000..29129cb4f --- /dev/null +++ b/tasks/run_graalpy_tests.py @@ -0,0 +1,121 @@ +"""Run pytest in parallel shards, tolerating GraalVM's polyglot interpreter-shutdown crash after a clean run.""" + +from __future__ import annotations + +import os +import subprocess +import sys +import threading +from argparse import REMAINDER, ArgumentParser +from pathlib import Path +from typing import IO, TYPE_CHECKING, Final +from xml.etree import ElementTree as ET # ruff:ignore[suspicious-xml-etree-import] junit report is generated by our own pytest run, not untrusted + +if TYPE_CHECKING: + from argparse import Namespace + from collections.abc import Sequence + +_SHUTDOWN: Final[str] = "did not complete all polyglot threads" +_NO_TESTS_COLLECTED: Final[int] = 5 + + +def main() -> None: + parser: Final[ArgumentParser] = ArgumentParser(description=__doc__) + parser.add_argument( + "--shards", + type=int, + default=os.cpu_count() or 1, + help="parallel pytest processes; xdist cannot run under GraalPy (see #3240)", + ) + parser.add_argument("junit", type=Path, help="report path; shard K writes it with a .K.xml suffix") + parser.add_argument("--group", default="1/1", help="K/N runner group; groups run disjoint shards") + parser.add_argument("pytest_args", nargs=REMAINDER, help="arguments passed to every pytest process") + args: Final[Namespace] = parser.parse_args() + group, _, groups = args.group.partition("/") + if not (group.isdecimal() and groups.isdecimal() and 1 <= int(group) <= int(groups)): + parser.error("--group requires K/N with 1 <= K <= N") + total: Final[int] = args.shards * int(groups) + shards: Final[list[_Shard]] = [ + _Shard(index, total, args.junit, args.pytest_args) for index in range(int(group), total + 1, int(groups)) + ] + executed: Final[list[int | None]] = [shard.wait() for shard in shards] + if failed := [shard.label for shard, count in zip(shards, executed) if count is None]: + msg = f"pytest shards failed: {', '.join(failed)}" + raise SystemExit(msg) + if not sum(filter(None, executed)): + msg = "pytest shards ran no tests" + raise SystemExit(msg) + + +class _Shard: + def __init__(self, index: int, count: int, junit: Path, pytest_args: Sequence[str]) -> None: + self.label: Final[str] = f"{index}/{count}" + self.junit: Final[Path] = junit.with_suffix(f".{index}.xml") + self.junit.unlink(missing_ok=True) + self.stderr: Final[list[str]] = [] + # -u streams output as it happens; the shards run for many minutes + self.process: Final[subprocess.Popen[str]] = subprocess.Popen( + [ + sys.executable, + "-u", + "-m", + "pytest", + *pytest_args, + f"--shard={self.label}", + "--junitxml", + str(self.junit), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + encoding="utf-8", + errors="replace", + ) + self.pumps: Final[list[threading.Thread]] = [ + threading.Thread(target=self._pump, args=(self.process.stdout, sys.stdout, None)), + threading.Thread(target=self._pump, args=(self.process.stderr, sys.stderr, self.stderr)), + ] + for pump in self.pumps: + pump.start() + + def _pump(self, source: IO[str], sink: IO[str], lines: list[str] | None) -> None: + for line in source: + if lines is not None: + lines.append(line) + sink.write(f"[{self.label}] {line}") + + def wait(self) -> int | None: + """Return how many tests ran, or None when the shard failed.""" + for pump in self.pumps: + pump.join() + status: Final[int] = self.process.wait() + shutdown: Final[bool] = status == 1 and any(_SHUTDOWN in line for line in self.stderr) + if status not in {0, _NO_TESTS_COLLECTED} and not shutdown: + print(f"[{self.label}] exit {status}") # ruff:ignore[print] + return None + if (executed := _executed_tests(self.junit)) is None: + print(f"[{self.label}] exit {status} without a clean junit report") # ruff:ignore[print] + return None + if shutdown: + print( # ruff:ignore[print] + f"[{self.label}] GraalVM reported its known polyglot shutdown error after a completed test run" + " (see #3240).", + ) + return executed + + +def _executed_tests(junit_path: Path) -> int | None: + try: + root: Final[ET.Element] = ET.parse(junit_path).getroot() # ruff:ignore[suspicious-xml-element-tree-usage] trusted pytest output + suites: Final[list[ET.Element]] = list(root.iter("testsuite")) + if not suites or any(suite.get("failures") != "0" or suite.get("errors") != "0" for suite in suites): + return None + return sum(int(suite.attrib["tests"]) - int(suite.get("skipped", "0")) for suite in suites) + except (OSError, ET.ParseError, KeyError, ValueError): + return None + + +__all__ = ["main"] + + +if __name__ == "__main__": + main() diff --git a/tasks/update_embedded.py b/tasks/update_embedded.py old mode 100755 new mode 100644 index 933e033f6..9d4254b12 --- a/tasks/update_embedded.py +++ b/tasks/update_embedded.py @@ -1,4 +1,4 @@ -"""Helper script to rebuild virtualenv.py from virtualenv_support.""" # noqa: EXE002 +"""Helper script to rebuild virtualenv.py from virtualenv_support.""" from __future__ import annotations @@ -6,10 +6,14 @@ import locale import os import re +from typing import TYPE_CHECKING, NoReturn from zlib import crc32 as _crc32 +if TYPE_CHECKING: + from pathlib import Path -def crc32(data): + +def crc32(data: str) -> int: """Python version idempotent.""" return _crc32(data.encode()) & 0xFFFFFFFF @@ -24,14 +28,17 @@ def crc32(data): file_template = '# file {filename}\n{variable} = convert(\n """\n{data}"""\n)' -def rebuild(script_path): - with script_path.open(encoding=locale.getpreferredencoding(False)) as current_fh: # noqa: FBT003 +def rebuild(script_path: Path) -> None: + encoding = ( + locale.getencoding() if hasattr(locale, "getencoding") else locale.getpreferredencoding(do_setlocale=False) + ) + with script_path.open(encoding=encoding) as current_fh: script_content = current_fh.read() script_parts = [] match_end = 0 next_match = None - _count, did_update = 0, False - for _count, next_match in enumerate(file_regex.finditer(script_content)): + count, did_update = 0, False + for count, next_match in enumerate(file_regex.finditer(script_content)): # ruff:ignore[unused-loop-control-variable] script_parts += [script_content[match_end : next_match.start()]] match_end = next_match.end() filename, variable_name, previous_encoded = next_match.group(1), next_match.group(2), next_match.group(3) @@ -43,11 +50,11 @@ def rebuild(script_path): script_parts += [script_content[match_end:]] new_content = "".join(script_parts) - report(1 if not _count or did_update else 0, new_content, next_match, script_content, script_path) + report(1 if not count or did_update else 0, new_content, next_match, script_content, script_path) -def handle_file(previous_content, filename, variable_name, previous_encoded): - print(f"Found file {filename}") # noqa: T201 +def handle_file(previous_content: str, filename: str, variable_name: str, previous_encoded: str) -> tuple[bool, str]: + print(f"Found file {filename}") # ruff:ignore[print] current_path = os.path.realpath(os.path.join(here, "..", "src", "virtualenv_embedded", filename)) _, file_type = os.path.splitext(current_path) keep_line_ending = file_type == ".bat" @@ -56,25 +63,25 @@ def handle_file(previous_content, filename, variable_name, previous_encoded): current_crc = crc32(current_text) current_encoded = b64.encode(gzip.encode(current_text.encode())[0])[0].decode() if current_encoded == previous_encoded: - print(f" File up to date (crc: {current_crc:08x})") # noqa: T201 + print(f" File up to date (crc: {current_crc:08x})") # ruff:ignore[print] return False, previous_content # Else: content has changed previous_text = gzip.decode(b64.decode(previous_encoded.encode())[0])[0].decode() previous_crc = crc32(previous_text) - print(f" Content changed (crc: {previous_crc:08x} -> {current_crc:08x})") # noqa: T201 + print(f" Content changed (crc: {previous_crc:08x} -> {current_crc:08x})") # ruff:ignore[print] new_part = file_template.format(filename=filename, variable=variable_name, data=current_encoded) return True, new_part -def report(exit_code, new, next_match, current, script_path): +def report(exit_code: int, new: str, next_match: re.Match[str] | None, current: str, script_path: Path) -> NoReturn: if new != current: - print("Content updated; overwriting... ", end="") # noqa: T201 + print("Content updated; overwriting... ", end="") # ruff:ignore[print] script_path.write_bytes(new) - print("done.") # noqa: T201 + print("done.") # ruff:ignore[print] else: - print("No changes in content") # noqa: T201 + print("No changes in content") # ruff:ignore[print] if next_match is None: - print("No variables were matched/found") # noqa: T201 + print("No variables were matched/found") # ruff:ignore[print] raise SystemExit(exit_code) diff --git a/tasks/upgrade_ci_tools.py b/tasks/upgrade_ci_tools.py new file mode 100644 index 000000000..4c8cbe63c --- /dev/null +++ b/tasks/upgrade_ci_tools.py @@ -0,0 +1,165 @@ +from __future__ import annotations + +import hashlib +import json +import os +import re +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import TYPE_CHECKING, Final +from urllib.request import Request, urlopen + +if TYPE_CHECKING: + from _hashlib import HASH + from typing import NotRequired, TypedDict + + class _Pin(TypedDict): + tag: NotRequired[str] + assets: NotRequired[dict[str, dict[str, str]]] + version: NotRequired[str] + + class _Release(TypedDict): + tagName: str + + class _Releases(TypedDict): + nodes: list[_Release] + + class _Repository(TypedDict): + latestRelease: _Release + releases: _Releases + + class _Asset(TypedDict): + name: str + id: int + digest: NotRequired[str | None] + + class _NpmVersion(TypedDict): + deprecated: NotRequired[str] + + class _NpmPackage(TypedDict): + time: dict[str, str] + versions: dict[str, _NpmVersion] + + +_PINS: Final[Path] = Path("tasks/ci-tools.json") +# no GraalPy assets to hash: check.yaml hands the tag to setup-python, which downloads the release itself +_TOOLS: Final[dict[str, tuple[str, bool, dict[str, str]]]] = { + "graalpy": ("oracle/graalpython", False, {}), + "nushell": ("nushell/nushell", False, {"Windows": "nu-{version}-x86_64-pc-windows-msvc.zip"}), + "rustpython": ( + "RustPython/RustPython", + True, + { + "Linux": "rustpython-release-Linux-x86_64-unknown-linux-gnu", + "macOS": "rustpython-release-macOS-aarch64-apple-darwin", + "Windows": "rustpython-release-Windows-x86_64-pc-windows-msvc.exe", + }, + ), +} +# the REST release listing of RustPython/RustPython comes back empty, so query GraphQL, which sees its releases +_QUERY: Final[str] = """ +query($owner: String!, $name: String!) { + repository(owner: $owner, name: $name) { + latestRelease { tagName } + releases(first: 1, orderBy: {field: CREATED_AT, direction: DESC}) { nodes { tagName } } + } +} +""" +# the docs load Mermaid from jsDelivr into every reader's browser, so give npm a week to pull a hijacked release +_NPM_COOLDOWN: Final[timedelta] = timedelta(days=7) + + +def main() -> None: + if not (token := os.environ.get("GH_TOKEN")): + msg = "GH_TOKEN is required to query GitHub releases" + raise SystemExit(msg) + pins: Final[dict[str, _Pin]] = json.loads(_PINS.read_text(encoding="utf-8")) + for tool, (repository, prerelease, patterns) in _TOOLS.items(): + # an unchanged tag keeps its recorded hashes, so an asset replaced under the same tag fails CI instead + if (tag := _newest_tag(token, repository, prerelease=prerelease)) != pins[tool]["tag"]: + pins[tool] = {"tag": tag, "assets": _pin_assets(token, repository, tag, patterns)} + pins["mermaid"]["version"] = _newest_npm_version("mermaid", pins["mermaid"]["version"]) + _PINS.write_text(f"{json.dumps(pins, indent=2, sort_keys=True)}\n", encoding="utf-8") + + +def _newest_tag(token: str, repository: str, *, prerelease: bool) -> str: + owner, name = repository.split("/") + with urlopen( + Request( + "https://api.github.com/graphql", + data=json.dumps({"query": _QUERY, "variables": {"owner": owner, "name": name}}).encode(), + headers={"Authorization": f"Bearer {token}"}, + ), + timeout=30, + ) as response: + found: Final[_Repository] = json.load(response)["data"]["repository"] + return found["releases"]["nodes"][0]["tagName"] if prerelease else found["latestRelease"]["tagName"] + + +def _pin_assets(token: str, repository: str, tag: str, patterns: dict[str, str]) -> dict[str, dict[str, str]]: + with urlopen( + Request( + f"https://api.github.com/repos/{repository}/releases/tags/{tag}", + headers={"Authorization": f"Bearer {token}"}, + ), + timeout=30, + ) as response: + published: Final[dict[str, _Asset]] = {asset["name"]: asset for asset in json.load(response)["assets"]} + pins: Final[dict[str, dict[str, str]]] = {} + for runner_os, pattern in patterns.items(): + if (asset := published.get(name := pattern.format(version=tag.removeprefix("v")))) is None: + msg = f"{repository} {tag} has no asset {name}" + raise SystemExit(msg) + sha256 = _sha256(repository, asset["id"]) + if (digest := asset.get("digest")) and digest != f"sha256:{sha256}": + msg = f"{name} hashes to {sha256} but GitHub reports {digest}" + raise SystemExit(msg) + pins[runner_os] = {"name": name, "sha256": sha256} + return pins + + +def _sha256(repository: str, asset_id: int) -> str: + digest: Final[HASH] = hashlib.sha256() + # no token: the API redirects to a signed storage URL, and urllib would forward the Authorization header there + with urlopen( + Request( + f"https://api.github.com/repos/{repository}/releases/assets/{asset_id}", + headers={"Accept": "application/octet-stream"}, + ), + timeout=300, + ) as response: + for chunk in iter(lambda: response.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _newest_npm_version(package: str, pinned: str) -> str: + with urlopen(Request(f"https://registry.npmjs.org/{package}"), timeout=30) as response: + metadata: Final[_NpmPackage] = json.load(response) + cutoff: Final[datetime] = datetime.now(tz=timezone.utc) - _NPM_COOLDOWN + # npm keeps the publish time of unpublished versions, so check each one still exists before taking it + return max( + ( + pinned, + *( + version + for version, published in metadata["time"].items() + if re.fullmatch(r"\d+\.\d+\.\d+", version) + and datetime.strptime(published, "%Y-%m-%dT%H:%M:%S.%f%z") <= cutoff + and version in metadata["versions"] + and not metadata["versions"][version].get("deprecated") + ), + ), + key=_release_key, + ) + + +def _release_key(version: str) -> tuple[int, ...]: + return tuple(map(int, version.split("."))) + + +__all__ = ["main"] + + +if __name__ == "__main__": + main() diff --git a/tasks/upgrade_wheels.py b/tasks/upgrade_wheels.py index 071e09b71..ccb338b59 100644 --- a/tasks/upgrade_wheels.py +++ b/tasks/upgrade_wheels.py @@ -2,151 +2,349 @@ from __future__ import annotations +import ast +import hashlib import os import shutil import subprocess import sys +import zipfile from collections import OrderedDict, defaultdict from pathlib import Path from tempfile import TemporaryDirectory from textwrap import dedent from threading import Thread +from typing import NoReturn STRICT = "UPGRADE_ADVISORY" not in os.environ -BUNDLED = ["pip", "setuptools", "wheel"] -SUPPORT = [(3, i) for i in range(8, 16)] +BUNDLED = ["pip", "setuptools"] +SUPPORT = [(3, i) for i in range(9, 17)] DEST = Path(__file__).resolve().parents[1] / "src" / "virtualenv" / "seed" / "wheels" / "embed" +NOTICES_DEST = Path(__file__).resolve().parents[1] / "THIRD-PARTY-NOTICES.md" -def download(ver, dest, package): - subprocess.call( - [ - sys.executable, - "-m", - "pip", - "--disable-pip-version-check", - "download", - "--no-cache-dir", - "--only-binary=:all:", - "--python-version", - ver, - "-d", - dest, - package, - ], - ) - - -def run(): # noqa: C901, PLR0912 +def run() -> NoReturn: + if "--regen" in sys.argv[1:]: + render_init() + render_notices() + raise SystemExit(0) old_batch = {i.name for i in DEST.iterdir() if i.suffix == ".whl"} with TemporaryDirectory() as temp: - temp_path = Path(temp) - folders = {} - targets = [] - for support in SUPPORT: - support_ver = ".".join(str(i) for i in support) - into = temp_path / support_ver - into.mkdir() - folders[into] = support_ver - for package in BUNDLED: - if package == "wheel" and support >= (3, 9): - continue - thread = Thread(target=download, args=(support_ver, str(into), package)) - targets.append(thread) - thread.start() - for thread in targets: - thread.join() + folders = _download_all(Path(temp)) new_batch = {i.name: i for f in folders for i in Path(f).iterdir()} - new_packages = new_batch.keys() - old_batch remove_packages = old_batch - new_batch.keys() - - for package in remove_packages: - (DEST / package).unlink() - for package in new_packages: - shutil.copy2(str(new_batch[package]), DEST / package) - + _sync_dest(new_packages, remove_packages, new_batch) added = collect_package_versions(new_packages) removed = collect_package_versions(remove_packages) outcome = (1 if STRICT else 0) if (added or removed) else 0 - print(f"Outcome {outcome} added {added} removed {removed}") # noqa: T201 - lines = ["Upgrade embedded wheels:", ""] - for key, versions in added.items(): - text = f"* {key} to {fmt_version(versions)}" - if key in removed: - rem = ", ".join(f"``{i}``" for i in removed[key]) - text += f" from {rem}" - del removed[key] - lines.append(text) - for key, versions in removed.items(): - lines.append(f"Removed {key} of {fmt_version(versions)}") - lines.append("") - changelog = "\n".join(lines) - print(changelog) # noqa: T201 - if len(lines) >= 4: # noqa: PLR2004 - (Path(__file__).parents[1] / "docs" / "changelog" / "u.bugfix.rst").write_text(changelog, encoding="utf-8") + print(f"Outcome {outcome} added {added} removed {removed}") # ruff:ignore[print] + _write_changelog(added, removed) + render_init(folders=folders) + render_notices() + raise SystemExit(outcome) + + +def _download_all(temp_path: Path) -> dict[Path, str]: + folders: dict[Path, str] = {} + targets: list[Thread] = [] + for support in SUPPORT: + support_ver = ".".join(str(i) for i in support) + into = temp_path / support_ver + into.mkdir() + folders[into] = support_ver + for package in BUNDLED: + thread = Thread(target=download, args=(support_ver, str(into), package)) + targets.append(thread) + thread.start() + for thread in targets: + thread.join() + return folders + + +def _sync_dest(new_packages: set[str], remove_packages: set[str], new_batch: dict[str, Path]) -> None: + for package in remove_packages: + (DEST / package).unlink() + for package in new_packages: + shutil.copy2(str(new_batch[package]), DEST / package) + + +def _write_changelog(added: dict[str, list[str]], removed: dict[str, list[str]]) -> None: + lines = ["Upgrade embedded wheels:", ""] + for key, versions in added.items(): + text = f"- {key} to {fmt_version(versions)}" + if key in removed: + rem = ", ".join(f"``{i}``" for i in removed[key]) + text += f" from {rem}" + del removed[key] + lines.append(text) + for key, versions in removed.items(): + lines.append(f"Removed {key} of {fmt_version(versions)}") + lines.append("") + changelog = "\n".join(lines) + print(changelog) # ruff:ignore[print] + if len(lines) >= 4: # ruff:ignore[magic-value-comparison] + (Path(__file__).parents[1] / "docs" / "changelog" / "u.bugfix.rst").write_text(changelog, encoding="utf-8") + + +def render_init(folders: dict[Path, str] | None = None) -> None: + """Write ``embed/__init__.py`` from the wheels currently in DEST. + + When called from ``run()`` after a download round, ``folders`` maps each per-python-version temp folder to its + version string, which is how support for a wheel is determined. When called with ``--regen`` there are no downloaded + folders — the existing ``BUNDLE_SUPPORT`` from the current ``__init__.py`` is used so regeneration is deterministic. + + """ + if folders is None: + support_table = _support_table_from_existing_init() + else: + present = {i.name: i for f in folders for i in Path(f).iterdir() if i.suffix == ".whl"} support_table = OrderedDict((".".join(str(j) for j in i), []) for i in SUPPORT) - for package in sorted(new_batch.keys()): + for package in sorted(present): for folder, version in sorted(folders.items()): if (folder / package).exists(): support_table[version].append(package) - support_table = {k: OrderedDict((i.split("-")[0], i) for i in v) for k, v in support_table.items()} - nl = "\n" - bundle = "".join( - f"\n {v!r}: {{{nl}{''.join(f' {p!r}: {f!r},{nl}' for p, f in line.items())} }}," - for v, line in support_table.items() - ) - msg = dedent( - f""" - from pathlib import Path + support_table = OrderedDict((k, OrderedDict((i.split("-")[0], i) for i in v)) for k, v in support_table.items()) + wheel_names = sorted({wheel for mapping in support_table.values() for wheel in mapping.values()}) + sha_table = OrderedDict((name, _sha256(DEST / name)) for name in wheel_names) + nl = "\n" + bundle = "".join( + f"\n {v!r}: {{{nl}{''.join(f' {p!r}: {f!r},{nl}' for p, f in line.items())} }}," + for v, line in support_table.items() + ) + sha_block = "".join(f"\n {name!r}: {digest!r}," for name, digest in sha_table.items()) + msg = dedent( + f""" + from __future__ import annotations + + import hashlib + import zipfile + from pathlib import Path + + from virtualenv.info import IS_ZIPAPP, ROOT + from virtualenv.seed.wheels.util import Wheel + + BUNDLE_FOLDER = Path(__file__).absolute().parent + BUNDLE_SUPPORT = {{ {bundle} }} + MAX = next(reversed(BUNDLE_SUPPORT)) + MIN = next(iter(BUNDLE_SUPPORT)) + + + def _release_tuple(version: str) -> tuple[int, ...]: + return tuple(int(part) for part in version.split(".")) + + + # oldest target Python version virtualenv still bundles seed wheels for; anything below this has no embedded pip + OLDEST_SUPPORTED = _release_tuple(MIN) - from virtualenv.seed.wheels.util import Wheel + # SHA-256 of every bundled wheel. Verified on load so a corrupted or tampered wheel on disk fails loud instead of + # being handed to pip. Generated together with ``BUNDLE_SUPPORT`` by ``tasks/upgrade_wheels.py``. + BUNDLE_SHA256 = {{ {sha_block} }} - BUNDLE_FOLDER = Path(__file__).absolute().parent - BUNDLE_SUPPORT = {{ {bundle} }} - MAX = {next(iter(support_table.keys()))!r} + _VERIFIED_WHEELS: set[str] = set() - def get_embed_wheel(distribution, for_py_version): - mapping = BUNDLE_SUPPORT.get(for_py_version, {{}}) or BUNDLE_SUPPORT[MAX] - wheel_file = mapping.get(distribution) - if wheel_file is None: - return None - path = BUNDLE_FOLDER / wheel_file - return Wheel.from_path(path) + def get_embed_wheel(distribution: str, for_py_version: str | None) -> Wheel | None: + \"\"\"Return the bundled wheel that ships with virtualenv for a given distribution and Python version. - __all__ = [ - "get_embed_wheel", - "BUNDLE_SUPPORT", - "MAX", - "BUNDLE_FOLDER", - ] + :param distribution: project name of the seed package, for example ``pip`` or ``setuptools``. + :param for_py_version: major.minor Python version string the environment will be created for, or ``None`` to use the + newest bundle. - """, + :returns: a :class:`Wheel` pointing at the verified bundled file, or ``None`` when no wheel is bundled for the + requested combination, including target versions below the oldest bundled one. + + :raises RuntimeError: if the bundled wheel on disk fails SHA-256 verification. + + \"\"\" + if for_py_version is None or _release_tuple(for_py_version) > _release_tuple(MAX): + # no specific target, or a Python newer than anything bundled: reuse the newest bundle + mapping = BUNDLE_SUPPORT[MAX] + else: # versions below the oldest bundled one fall through to None instead of an incompatible newer wheel + mapping = BUNDLE_SUPPORT.get(for_py_version) + if not mapping: + return None + wheel_file = mapping.get(distribution) + if wheel_file is None: + return None + path = BUNDLE_FOLDER / wheel_file + _verify_bundled_wheel(path) + return Wheel.from_path(path) + + + def _verify_bundled_wheel(path: Path) -> None: + name = path.name + if name in _VERIFIED_WHEELS: + return + expected = BUNDLE_SHA256.get(name) + if expected is None: + msg = f"bundled wheel {{name}} has no recorded sha256 in BUNDLE_SHA256" + raise RuntimeError(msg) + actual = _hash_bundled_wheel(path) + if actual != expected: + msg = f"bundled wheel {{name}} sha256 mismatch: expected {{expected}}, got {{actual}}" + raise RuntimeError(msg) + _VERIFIED_WHEELS.add(name) + + + def _hash_bundled_wheel(path: Path) -> str: + # ``path`` is under the package directory; when virtualenv runs from a zipapp the wheel lives inside the + # archive and cannot be opened as a regular file, so read the bytes straight from the zipapp entry. + digest = hashlib.sha256() + if IS_ZIPAPP: + entry = path.resolve().relative_to(Path(ROOT).resolve()).as_posix() + with zipfile.ZipFile(ROOT, "r") as archive, archive.open(entry) as stream: + for chunk in iter(lambda: stream.read(1 << 20), b""): + digest.update(chunk) + else: + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + + __all__ = [ + "BUNDLE_FOLDER", + "BUNDLE_SHA256", + "BUNDLE_SUPPORT", + "MAX", + "MIN", + "OLDEST_SUPPORTED", + "get_embed_wheel", + ] + + """, + ) + dest_target = DEST / "__init__.py" + dest_target.write_text(msg, encoding="utf-8") + subprocess.run([sys.executable, "-m", "ruff", "check", str(dest_target), "--fix", "--unsafe-fixes"], check=False) + subprocess.run([sys.executable, "-m", "ruff", "format", str(dest_target), "--preview"], check=False) + + +def render_notices() -> None: + """Write ``THIRD-PARTY-NOTICES.md`` from the wheels currently bundled in DEST. + + Reads the ``BUNDLE_SHA256`` table ``render_init`` just wrote, so the notices always match what is actually bundled. + Only the license of the wheel virtualenv itself redistributes is reproduced here - whatever that wheel in turn + vendors (pip's own ``_vendor`` tree, for example) already carries its license inside the wheel and stays there + rather than being duplicated at the top level. + + """ + tree = ast.parse((DEST / "__init__.py").read_text(encoding="utf-8")) + wheel_names: list[str] = [] + for node in tree.body: + if isinstance(node, ast.Assign) and any( + isinstance(target, ast.Name) and target.id == "BUNDLE_SHA256" for target in node.targets + ): + wheel_names = sorted(ast.literal_eval(node.value)) + break + groups: dict[tuple[str, str], list[str]] = defaultdict(list) + for wheel_name in wheel_names: + distribution = wheel_name.split("-")[0] + groups[distribution, _wheel_license_text(DEST / wheel_name)].append(wheel_name) + sections = [] + for (distribution, text), names in sorted(groups.items()): + filenames = ", ".join(f"``{name}``" for name in sorted(names)) + sections.append( + f"## {distribution}\n\nBundled as: {filenames}\n\n" + f"Upstream: https://pypi.org/project/{distribution}/\n\n" + f"```\n{text.strip()}\n```\n" ) - dest_target = DEST / "__init__.py" - dest_target.write_text(msg, encoding="utf-8") - subprocess.run([sys.executable, "-m", "ruff", "check", str(dest_target), "--fix", "--unsafe-fixes"]) - subprocess.run([sys.executable, "-m", "ruff", "format", str(dest_target), "--preview"]) + header = dedent("""\ + + # Third-party notices + + virtualenv is distributed under the MIT License, see `LICENSE`. The wheels under + `src/virtualenv/seed/wheels/embed/` are redistributed unmodified and remain under their own licenses, + reproduced below. Each wheel also carries its own license text inside its `.dist-info/licenses/` + directory, including the licenses of anything it in turn vendors. + + """) + NOTICES_DEST.write_text(header + "\n".join(sections), encoding="utf-8") + if (pre_commit := shutil.which("pre-commit")) is None: + msg = "pre-commit must be installed to format THIRD-PARTY-NOTICES.md" + raise RuntimeError(msg) + command = [pre_commit, "run", "mdformat", "--files", str(NOTICES_DEST)] + subprocess.run(command, check=False) + subprocess.run(command, check=True) + + +def _wheel_license_text(path: Path) -> str: + with zipfile.ZipFile(path) as archive: + names = archive.namelist() + dist_info = next(name.split("/", 1)[0] for name in names if name.endswith(".dist-info/METADATA")) + prefix = f"{dist_info}/licenses/" + # top-level files only: a nested match would be a license belonging to something the wheel itself vendors + candidates = sorted( + name + for name in names + if name.startswith(prefix) and "/" not in name[len(prefix) :] and "LICENSE" in Path(name).name.upper() + ) + if not candidates: + msg = f"no top-level license file found under {prefix} in {path.name}" + raise RuntimeError(msg) + return archive.read(candidates[0]).decode("utf-8") - raise SystemExit(outcome) +def _support_table_from_existing_init() -> OrderedDict[str, OrderedDict[str, str]]: + source = (DEST / "__init__.py").read_text(encoding="utf-8") + tree = ast.parse(source) + for node in tree.body: + if isinstance(node, ast.Assign) and any( + isinstance(t, ast.Name) and t.id == "BUNDLE_SUPPORT" for t in node.targets + ): + bundle_support = ast.literal_eval(node.value) + return OrderedDict( + (version, OrderedDict(sorted(mapping.items()))) for version, mapping in bundle_support.items() + ) + msg = f"BUNDLE_SUPPORT not found in {DEST / '__init__.py'}" + raise RuntimeError(msg) -def fmt_version(versions): + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def fmt_version(versions: list[str]) -> str: return ", ".join(f"``{v}``" for v in versions) -def collect_package_versions(new_packages): +def collect_package_versions(new_packages: set[str]) -> dict[str, list[str]]: result = defaultdict(list) for package in new_packages: split = package.split("-") - if len(split) < 2: # noqa: PLR2004 + if len(split) < 2: # ruff:ignore[magic-value-comparison] raise ValueError(package) key, version = split[0:2] result[key].append(version) return result +def download(python_version: str, dest: str, package: str) -> None: + subprocess.call( + [ + sys.executable, + "-W", + "ignore::EncodingWarning", + "-m", + "pip", + "--disable-pip-version-check", + "download", + "--no-cache-dir", + "--only-binary=:all:", + "--python-version", + python_version, + "-d", + dest, + package, + ], + ) + + if __name__ == "__main__": run() diff --git a/tasks/validate_sbom.py b/tasks/validate_sbom.py new file mode 100644 index 000000000..bd0e503f2 --- /dev/null +++ b/tasks/validate_sbom.py @@ -0,0 +1,187 @@ +"""Check that the SBOM in a built wheel or zipapp is valid CycloneDX 1.6, satisfies actions/attest and license policy.""" + +from __future__ import annotations + +import hashlib +import json +import re +import sys +import zipfile +from pathlib import Path +from typing import TYPE_CHECKING, Any, Final + +from cyclonedx.schema import SchemaVersion +from cyclonedx.validation.json import JsonStrictValidator +from cyclonedx_to_spdx import to_spdx +from license_policy import license_problems +from spdx_tools.spdx.parser.error import SPDXParsingError +from spdx_tools.spdx.parser.jsonlikedict.json_like_dict_parser import JsonLikeDictParser +from spdx_tools.spdx.validation.document_validator import validate_full_spdx_document + +if TYPE_CHECKING: + from collections.abc import Iterator + from typing import NotRequired, TypedDict + + class _Hash(TypedDict): + alg: str + content: str + + class _Property(TypedDict): + name: str + value: str + + class _Component(TypedDict): + type: str + name: str + hashes: NotRequired[list[_Hash]] + properties: NotRequired[list[_Property]] + components: NotRequired[list[_Component]] + + from cyclonedx_to_spdx import CycloneDX + +_SERIAL_PATTERN: Final[re.Pattern[str]] = re.compile( + r""" + ^urn:uuid: + (?P[0-9a-f]{8})- + (?P[0-9a-f]{4})- + (?P[0-9a-f]{4})- + (?P[0-9a-f]{4})- + (?P[0-9a-f]{12}) + $ + """, + re.VERBOSE, +) +# tasks/zipapp_sbom.py writes it at the archive root, next to the zipapp's __main__.py +_ZIPAPP_SBOM: Final[str] = "virtualenv.pyz.cdx.json" + + +def main() -> None: + target = Path(sys.argv[1]) + if target.suffix == ".pyz": + artifact, problems = target, validate_zipapp(target) + else: + wheels = sorted(target.glob("*.whl")) + if len(wheels) != 1: + msg = f"expected exactly one wheel in {target}, found {len(wheels)}: {[w.name for w in wheels]}" + raise SystemExit(msg) + artifact, problems = wheels[0], validate(wheels[0]) + if problems: + for problem in problems: + print(f"SBOM invalid: {problem}") # ruff:ignore[print] + raise SystemExit(1) + print(f"SBOM in {artifact.name} is valid") # ruff:ignore[print] + + +def validate(wheel: Path) -> list[str]: + with zipfile.ZipFile(wheel) as archive: + sbom_name = next((n for n in archive.namelist() if n.endswith("/sboms/virtualenv.cdx.json")), None) + if sbom_name is None: + return ["no .dist-info/sboms/virtualenv.cdx.json found in the wheel"] + raw = archive.read(sbom_name).decode("utf-8") + + if schema_error := JsonStrictValidator(SchemaVersion.V1_6).validate_str(raw): + return [f"does not conform to the CycloneDX 1.6 schema: {schema_error}"] + document = json.loads(raw) + + problems = _validate_serial(document.get("serialNumber", "")) + problems += _validate_references(document) + problems += _validate_spdx(document) + problems += license_problems(document["components"]) + return problems + + +def validate_zipapp(pyz: Path) -> list[str]: + with zipfile.ZipFile(pyz) as archive: + if _ZIPAPP_SBOM not in archive.namelist(): + return [f"no {_ZIPAPP_SBOM} found in {pyz.name}"] + raw = archive.read(_ZIPAPP_SBOM).decode("utf-8") + contents: Final[dict[str, str]] = { + name: hashlib.sha256(archive.read(name)).hexdigest() + for name in archive.namelist() + if not name.endswith("/") and name != _ZIPAPP_SBOM + } + if schema_error := JsonStrictValidator(SchemaVersion.V1_6).validate_str(raw): + return [f"does not conform to the CycloneDX 1.6 schema: {schema_error}"] + document = json.loads(raw) + problems = _validate_serial(document.get("serialNumber", "")) + problems += _validate_references(document) + problems += license_problems(document["components"]) + described: Final[dict[str, str]] = dict( + _described_files([document["metadata"]["component"], *document["components"]]) + ) + if missing := sorted(contents.keys() - described.keys()): + problems.append(f"{pyz.name} holds files the SBOM does not describe: {missing}") + if unknown := sorted(described.keys() - contents.keys()): + problems.append(f"SBOM describes files {pyz.name} does not hold: {unknown}") + if changed := sorted(name for name in contents.keys() & described.keys() if contents[name] != described[name]): + problems.append(f"SHA-256 in the SBOM differs from the archive for: {changed}") + return problems + + +def _validate_serial(serial: str) -> list[str]: + # optional in the CycloneDX spec itself, but actions/attest's format sniffer requires it to recognize + # the document as CycloneDX at all, and silently rejects anything missing it as an unknown format + return [] if _SERIAL_PATTERN.match(serial) else [f"serialNumber must be a urn:uuid: RFC 4122 UUID, got {serial!r}"] + + +def _validate_references(document: dict[str, Any]) -> list[str]: + root_ref = document["metadata"]["component"]["bom-ref"] + component_refs = {component["bom-ref"] for component in document["components"]} + tool_refs = {tool["bom-ref"] for tool in document["metadata"]["tools"]["components"]} + every_ref = list( + _bom_refs([ + document["metadata"]["component"], + *document["components"], + *document["metadata"]["tools"]["components"], + ]) + ) + if duplicates := {ref for ref in every_ref if every_ref.count(ref) > 1}: + return [f"bom-ref values are not unique: {sorted(duplicates)}"] + known = set(every_ref) + + problems = [] + dependencies = {entry["ref"]: set(entry.get("dependsOn", [])) for entry in document["dependencies"]} + if unknown := (set(dependencies) | set().union(*dependencies.values())) - known: + problems.append(f"dependencies reference bom-refs that do not exist: {sorted(unknown)}") + if dependencies.get(root_ref) != component_refs: + problems.append( + f"root dependsOn {sorted(dependencies.get(root_ref, []))} != components {sorted(component_refs)}" + ) + + for workflow in document["formulation"][0]["workflows"]: + referenced = {reference["ref"] for reference in workflow["resourceReferences"]} + if unknown := referenced - tool_refs: + problems.append(f"workflow {workflow['uid']} references unknown tools: {sorted(unknown)}") + return problems + + +def _validate_spdx(document: CycloneDX) -> list[str]: + try: + spdx = JsonLikeDictParser().parse(dict(to_spdx(document))) + except SPDXParsingError as error: + return [f"SPDX rendering does not parse: {message}" for message in error.get_messages()] + return [f"SPDX rendering invalid: {message.validation_message}" for message in validate_full_spdx_document(spdx)] + + +def _described_files(components: list[_Component]) -> Iterator[tuple[str, str]]: + for component in components: + digest = next((entry["content"] for entry in component.get("hashes", []) if entry["alg"] == "SHA-256"), "") + if component["type"] == "file": + yield component["name"], digest + # an embedded wheel is a library component whose bytes are one archive member + yield from ( + (prop["value"], digest) + for prop in component.get("properties", []) + if prop["name"] == "virtualenv:bundled-wheel" + ) + yield from _described_files(component.get("components", [])) + + +def _bom_refs(components: list[dict[str, Any]]) -> Iterator[str]: + for component in components: + yield component["bom-ref"] + yield from _bom_refs(component.get("components", [])) + + +if __name__ == "__main__": + main() diff --git a/tasks/zipapp_sbom.py b/tasks/zipapp_sbom.py new file mode 100644 index 000000000..b1c9e2b20 --- /dev/null +++ b/tasks/zipapp_sbom.py @@ -0,0 +1,204 @@ +"""Describe what a built virtualenv zipapp contains as a CycloneDX SBOM, and embed that SBOM in the zipapp.""" + +from __future__ import annotations + +import hashlib +import json +import sys +import uuid +import zipfile +from datetime import datetime, timezone +from importlib.metadata import PathDistribution +from io import BytesIO +from pathlib import Path, PurePosixPath +from typing import TYPE_CHECKING, Final +from urllib.parse import quote + +from hatchling.builders.utils import get_reproducible_timestamp + +from hatch_build import PYPA, SBOM_NAMESPACE, build_tools, component_from_metadata, timestamp + +if TYPE_CHECKING: + from importlib.metadata import PackageMetadata + + from hatch_build import Body, Component, Document, Property + +_SBOM_NAME: Final[str] = "virtualenv.pyz.cdx.json" +_HERE: Final[Path] = Path(__file__).parent +_RELEASES: Final[str] = "https://github.com/pypa/virtualenv/releases/download" +_BUNDLED: Final[str] = "__virtualenv__/" +_EMBEDDED_WHEELS: Final[str] = "virtualenv/seed/wheels/embed/" +# written by tasks/make_zipapp.py to find modules and metadata of the bundled distributions at run time +_LOADER: Final[frozenset[str]] = frozenset({"__main__.py", "distributions.json", "modules.json"}) + + +def main() -> None: + pyz, target = map(Path, sys.argv[1:3]) + with zipfile.ZipFile(pyz) as archive: + content: Final[str] = _describe(archive) + target.write_text(content, encoding="utf-8") + entry: Final[zipfile.ZipInfo] = zipfile.ZipInfo( + _SBOM_NAME, datetime.fromtimestamp(get_reproducible_timestamp(), tz=timezone.utc).timetuple()[:6] + ) + # match the entries tasks/make_zipapp.py writes, whichever OS appends the SBOM + entry.create_system = 3 + entry.external_attr = 0o644 << 16 + with zipfile.ZipFile(pyz, "a") as archive: + archive.writestr(entry, content) + + +def _describe(archive: zipfile.ZipFile) -> str: + members: Final[list[str]] = sorted(name for name in archive.namelist() if not name.endswith("/")) + metadata: Final[PackageMetadata] = _metadata( + archive, next(name for name in members if name.count("/") == 1 and name.endswith(".dist-info/METADATA")) + ) + version: Final[str] = metadata["Version"] + purl: Final[str] = f"pkg:generic/virtualenv.pyz@{quote(version, safe='')}" + root: Final[Component] = { + "type": "application", + "bom-ref": purl, + "supplier": PYPA, + "name": "virtualenv.pyz", + "version": version, + "description": "virtualenv and its runtime dependencies for every supported Python, as one zipapp", + "purl": purl, + "externalReferences": [ + {"type": "distribution", "url": f"{_RELEASES}/{version}/virtualenv.pyz"}, + {"type": "distribution", "url": "https://bootstrap.pypa.io/virtualenv.pyz", "comment": "latest release"}, + ], + "components": [_file(archive, purl, name) for name in members if name in _LOADER], + } + virtualenv: Final[Component] = _from_metadata(metadata) + # the wheel SBOM in virtualenv's dist-info already lists what each embedded wheel vendors + virtualenv["components"] = [ + _embedded_wheel(archive, name) + if name.startswith(_EMBEDDED_WHEELS) and name.endswith(".whl") + else _file(archive, virtualenv["bom-ref"], name) + for name in members + if not name.startswith(_BUNDLED) and name not in _LOADER and name != _SBOM_NAME + ] + loaded_by: Final[dict[str, list[str]]] = {} + for python, platforms in json.loads(archive.read("distributions.json")).items(): + for paths in platforms.values(): + for path in paths.values(): + loaded_by.setdefault(str(PurePosixPath(path).parent), []).append(python) + bundled: Final[list[Component]] = [ + _bundled(archive, directory, [name for name in members if name.startswith(f"{directory}/")], pythons) + for directory, pythons in sorted(loaded_by.items()) + ] + environment, tool_dependencies = build_tools(version) + tools: Final[list[Component]] = [ + *environment, + *(_script(path, version) for path in (_HERE / "make_zipapp.py", Path(__file__))), + ] + body: Final[Body] = { + "metadata": { + "timestamp": timestamp(), + "lifecycles": [{"phase": "build"}], + "tools": {"components": tools}, + "manufacturer": PYPA, + "supplier": PYPA, + "component": root, + }, + "components": [virtualenv, *bundled], + "dependencies": [ + {"ref": purl, "dependsOn": [virtualenv["bom-ref"], *(component["bom-ref"] for component in bundled)]}, + *tool_dependencies, + ], + "formulation": [ + { + "bom-ref": "formula:zipapp-build", + "workflows": [ + { + "bom-ref": "workflow:zipapp-build", + "uid": "zipapp-build", + "name": "build the zipapp and this SBOM", + "taskTypes": ["build"], + "resourceReferences": [{"ref": tool["bom-ref"]} for tool in tools], + "inputs": [ + { + "environmentVars": [ + {"name": "SOURCE_DATE_EPOCH", "value": str(get_reproducible_timestamp())} + ] + } + ], + "outputs": [{"type": "artifact", "resource": {"ref": purl}}], + } + ], + } + ], + } + document: Final[Document] = { + "$schema": "http://cyclonedx.org/schema/bom-1.6.schema.json", + "bomFormat": "CycloneDX", + "specVersion": "1.6", + # derived from the content so identical documents share a serial and any difference gets a new one + "serialNumber": f"urn:uuid:{uuid.uuid5(SBOM_NAMESPACE, json.dumps(body, sort_keys=True))}", + "version": 1, + **body, + } + return json.dumps(document, indent=2) + "\n" + + +def _metadata(archive: zipfile.ZipFile, name: str) -> PackageMetadata: + return PathDistribution(zipfile.Path(archive, f"{PurePosixPath(name).parent}/")).metadata + + +def _from_metadata(metadata: PackageMetadata) -> Component: + return component_from_metadata(metadata, "library") + + +def _file(archive: zipfile.ZipFile, parent: str, name: str) -> Component: + content: Final[bytes] = archive.read(name) + return { + "type": "file", + "bom-ref": f"{parent}#{name}", + "name": name, + "hashes": [{"alg": "SHA-256", "content": hashlib.sha256(content).hexdigest()}], + "properties": [{"name": "size", "value": str(len(content))}], + } + + +def _embedded_wheel(archive: zipfile.ZipFile, name: str) -> Component: + content: Final[bytes] = archive.read(name) + with zipfile.ZipFile(BytesIO(content)) as wheel: + metadata: Final[PackageMetadata] = _metadata( + wheel, next(entry for entry in wheel.namelist() if entry.endswith(".dist-info/METADATA")) + ) + component: Final[Component] = _from_metadata(metadata) + component["hashes"] = [{"alg": "SHA-256", "content": hashlib.sha256(content).hexdigest()}] + component["properties"] = [{"name": "virtualenv:bundled-wheel", "value": name}, *component["properties"]] + return component + + +def _bundled(archive: zipfile.ZipFile, directory: str, names: list[str], pythons: list[str]) -> Component: + component: Final[Component] = _from_metadata( + _metadata(archive, next(name for name in names if name.endswith(".dist-info/METADATA"))) + ) + loaded_for: Final[list[Property]] = [ + {"name": "virtualenv:loaded-for-python", "value": python} for python in pythons + ] + component["properties"] = [ + {"name": "virtualenv:zipapp-directory", "value": directory}, + *loaded_for, + *component["properties"], + ] + component["components"] = [_file(archive, component["bom-ref"], name) for name in names] + return component + + +def _script(path: Path, version: str) -> Component: + return { + "type": "application", + "bom-ref": f"tool:{path.name}", + "name": path.name, + "version": version, + "hashes": [{"alg": "SHA-256", "content": hashlib.sha256(path.read_bytes()).hexdigest()}], + "externalReferences": [ + {"type": "vcs", "url": f"https://github.com/pypa/virtualenv/blob/main/tasks/{path.name}"} + ], + } + + +if __name__ == "__main__": + main() diff --git a/tests/conftest.py b/tests/conftest.py index e4fc28479..4221c0390 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,25 +4,43 @@ import os import shutil import sys +from argparse import ArgumentTypeError from contextlib import contextmanager from functools import partial from pathlib import Path from typing import ClassVar import pytest +from python_discovery import PythonInfo from virtualenv.app_data import AppDataDiskFolder -from virtualenv.discovery.py_info import PythonInfo -from virtualenv.info import IS_GRAALPY, IS_PYPY, IS_WIN, fs_supports_symlink +from virtualenv.info import IS_GRAALPY, IS_PYPY, IS_RUSTPYTHON, IS_WIN, fs_supports_symlink from virtualenv.report import LOGGER +# the build and CI tooling tests do not depend on the interpreter; the tasks tox environment runs them once +collect_ignore = ["tasks"] -def pytest_addoption(parser): + +def pytest_addoption(parser) -> None: parser.addoption("--int", action="store_true", default=False, help="run integration tests") parser.addoption("--skip-slow", action="store_true", default=False, help="skip slow tests") + parser.addoption( + "--shard", + type=_parse_shard, + metavar="K/N", + help="run the K-th of N disjoint slices of the collected tests", + ) + +def _parse_shard(value: str) -> tuple[int, int]: + index, _, count = value.partition("/") + if not (index.isdecimal() and count.isdecimal() and 1 <= int(index) <= int(count)): + msg = f"expected K/N with 1 <= K <= N, got {value!r}" + raise ArgumentTypeError(msg) + return int(index), int(count) -def pytest_configure(config): + +def pytest_configure(config) -> None: """Ensure randomly is called before we re-order""" manager = config.pluginmanager @@ -35,7 +53,13 @@ def pytest_configure(config): order[from_pos] = temp -def pytest_collection_modifyitems(config, items): +def pytest_collection_modifyitems(config, items) -> None: + if shard := config.getoption("--shard"): + index, count = shard + # pytest-randomly shuffles each process differently, so slice the sorted IDs to agree across shards + selected = set(sorted(item.nodeid for item in items)[index - 1 :: count]) + config.hook.pytest_deselected(items=[item for item in items if item.nodeid not in selected]) + items[:] = [item for item in items if item.nodeid in selected] int_location = os.path.join("tests", "integration", "").rstrip() if len(items) == 1: return @@ -54,7 +78,7 @@ def pytest_collection_modifyitems(config, items): @pytest.fixture(scope="session") -def has_symlink_support(tmp_path_factory): # noqa: ARG001 +def has_symlink_support(): return fs_supports_symlink() @@ -64,7 +88,7 @@ def link_folder(has_symlink_support): return os.symlink if sys.platform == "win32": # on Windows junctions may be used instead - import _winapi # noqa: PLC0415 + import _winapi # ruff:ignore[import-outside-top-level] return getattr(_winapi, "CreateJunction", None) return None @@ -148,12 +172,30 @@ def _ignore_global_config(tmp_path_factory): yield +def _assert_environ_stable(old: dict[str, str], to_clean: set[str]) -> None: + new = os.environ + extra = {k: new[k] for k in set(new) - set(old)} + miss = {k: old[k] for k in set(old) - set(new) - to_clean} + diff = { + f"{k} = {old[k]} vs {new[k]}" for k in set(old) & set(new) if old[k] != new[k] and not k.startswith("PYTEST_") + } + if extra or miss or diff: + msg = "tests changed environ" + if extra: + msg += f" extra {extra}" + if miss: + msg += f" miss {miss}" + if diff: + msg += f" diff {diff}" + pytest.fail(msg) + + @pytest.fixture(autouse=True) def _check_os_environ_stable(): old = os.environ.copy() # ensure we don't inherit parent env variables to_clean = {k for k in os.environ if k.startswith(("VIRTUALENV_", "TOX_")) or "VIRTUAL_ENV" in k} - cleaned = {k: os.environ[k] for k, v in os.environ.items()} + cleaned = {k: os.environ[k] for k in os.environ} override = { "VIRTUALENV_NO_PERIODIC_UPDATE": "1", "VIRTUALENV_NO_DOWNLOAD": "1", @@ -171,23 +213,7 @@ def _check_os_environ_stable(): for key in override: del os.environ[str(key)] if is_exception is False: - new = os.environ - extra = {k: new[k] for k in set(new) - set(old)} - miss = {k: old[k] for k in set(old) - set(new) - to_clean} - diff = { - f"{k} = {old[k]} vs {new[k]}" - for k in set(old) & set(new) - if old[k] != new[k] and not k.startswith("PYTEST_") - } - if extra or miss or diff: - msg = "tests changed environ" - if extra: - msg += f" extra {extra}" - if miss: - msg += f" miss {miss}" - if diff: - msg += f" diff {diff}" - pytest.fail(msg) + _assert_environ_stable(old, to_clean) finally: os.environ.update(cleaned) @@ -198,12 +224,10 @@ def _check_os_environ_stable(): @pytest.fixture(autouse=True) def coverage_env(monkeypatch, link, request): - """ - Enable coverage report collection on the created virtual environments by injecting the coverage project - """ + """Enable coverage report collection on the created virtual environments by injecting the coverage project""" if COVERAGE_RUN and "_no_coverage" not in request.fixturenames: # we inject right after creation, we cannot collect coverage on site.py - used for helper scripts, such as debug - from virtualenv import run # noqa: PLC0415 + from virtualenv import run # ruff:ignore[import-outside-top-level] def _session_via_cli(args, options, setup_logging, env=None): session = prev_run(args, options, setup_logging, env) @@ -212,7 +236,7 @@ def _session_via_cli(args, options, setup_logging, env=None): def create_run(): result = old_run() obj["cov"] = EnableCoverage(link) - obj["cov"].__enter__(session.creator) # noqa: PLC2801 + obj["cov"].__enter__(session.creator) # ruff:ignore[unnecessary-dunder-call] return result monkeypatch.setattr(session.creator, "run", create_run) @@ -222,7 +246,7 @@ def create_run(): prev_run = run.session_via_cli monkeypatch.setattr(run, "session_via_cli", _session_via_cli) - def finish(): + def finish() -> None: cov = obj["cov"] obj["cov"] = None cov.__exit__(None, None, None) @@ -233,7 +257,7 @@ def finish(): else: - def finish(): + def finish() -> None: pass yield finish @@ -241,7 +265,7 @@ def finish(): # _no_coverage tells coverage_env to disable coverage injection for _no_coverage user. @pytest.fixture -def _no_coverage(): +def _no_coverage() -> None: pass @@ -258,7 +282,7 @@ def __init__(self, link) -> None: self.link = link self.targets = [] - def __enter__(self, creator): # noqa: PLE0302 + def __enter__(self, creator): # ruff:ignore[unexpected-special-method-signature] site_packages = creator.purelib for entry in self._ROOT_COV_FILES_AND_FOLDERS: target = site_packages / entry.name @@ -281,11 +305,12 @@ def is_inside_ci(): @pytest.fixture(scope="session") def special_char_name(): - base = "'\";&&e-$ èрт🚒♞中片-j" + # __VIRTUAL_NAME__ is a template placeholder, and must stay data when the path embeds it + base = "'\";&&e-$ !%`|>__VIRTUAL_NAME__èрт🚒♞中片-j" if IS_WIN: - # get rid of invalid characters on Windows - base = base.replace('"', "") - base = base.replace(";", "") + # NTFS path components cannot contain any of these, regardless of where they sit in the string + for reserved in '<>:"/\\|?*;': + base = base.replace(reserved, "") # workaround for pypy3 https://bitbucket.org/pypy/pypy/issues/3147/venv-non-ascii-support-windows encoding = "ascii" if IS_WIN else sys.getfilesystemencoding() # let's not include characters that the file system cannot encode) @@ -295,7 +320,7 @@ def special_char_name(): trip = char.encode(encoding, errors="strict").decode(encoding) if char == trip: result += char - except ValueError: # noqa: PERF203 + except ValueError: # ruff:ignore[try-except-in-loop] continue assert result return result @@ -308,7 +333,9 @@ def special_name_dir(tmp_path, special_char_name): @pytest.fixture(scope="session") def current_creators(session_app_data): - return PythonInfo.current_system(session_app_data).creators() + from virtualenv.run.plugin.creators import CreatorSelector # ruff:ignore[import-outside-top-level] + + return CreatorSelector.for_interpreter(PythonInfo.current_system(session_app_data)) @pytest.fixture(scope="session") @@ -326,10 +353,7 @@ def session_app_data(tmp_path_factory): @contextmanager def change_env_var(key, value): - """Temporarily change an environment variable. - :param key: the key of the env var - :param value: the value of the env var - """ + """Temporarily change an environment variable. :param key: the key of the env var :param value: the value of the env var""" already_set = key in os.environ prev_value = os.environ.get(key) os.environ[key] = value @@ -350,18 +374,18 @@ def temp_app_data(monkeypatch, tmp_path): @pytest.fixture(scope="session") -def for_py_version(): +def for_py_version() -> str: return f"{sys.version_info.major}.{sys.version_info.minor}" @pytest.fixture -def _skip_if_test_in_system(session_app_data): +def _skip_if_test_in_system(session_app_data) -> None: current = PythonInfo.current(session_app_data) if current.system_executable is not None: pytest.skip("test not valid if run under system") -if IS_PYPY or IS_GRAALPY: +if IS_PYPY or IS_GRAALPY or IS_RUSTPYTHON: @pytest.fixture def time_freeze(freezer): diff --git a/tests/integration/test_race_condition_simulation.py b/tests/integration/test_race_condition_simulation.py index 857de9aa5..0aec184f8 100644 --- a/tests/integration/test_race_condition_simulation.py +++ b/tests/integration/test_race_condition_simulation.py @@ -6,15 +6,23 @@ from pathlib import Path -def test_race_condition_simulation(tmp_path): +def _check_distutils_patch(errors: list[str]) -> None: + try: + importlib.util.find_spec("distutils.dist") + except NameError as e: + if "_DISTUTILS_PATCH" in str(e): + errors.append(str(e)) + + +def test_race_condition_simulation(tmp_path) -> None: """Test that simulates the race condition described in the issue. - This test creates a temporary directory with _virtualenv.py and _virtualenv.pth, - then simulates the scenario where: - - One process imports and uses the _virtualenv module (simulating marimo) - - Another process overwrites the _virtualenv.py file (simulating uv venv) + This test creates a temporary directory with _virtualenv.py and _virtualenv.pth, then simulates the scenario where: + - One process imports and uses the _virtualenv module (simulating marimo) - Another process overwrites the + _virtualenv.py file (simulating uv venv) The test verifies that no NameError is raised for _DISTUTILS_PATCH. + """ # Create the _virtualenv.py file virtualenv_file = tmp_path / "_virtualenv.py" @@ -29,20 +37,12 @@ def test_race_condition_simulation(tmp_path): # Simulate the race condition by repeatedly importing errors = [] for _ in range(5): - # Try to import it sys.path.insert(0, str(tmp_path)) + sys.modules.pop("_virtualenv", None) try: - if "_virtualenv" in sys.modules: - del sys.modules["_virtualenv"] - - import _virtualenv # noqa: F401, PLC0415 + import _virtualenv # ruff:ignore[unused-import, import-outside-top-level] - # Try to trigger find_spec - try: - importlib.util.find_spec("distutils.dist") - except NameError as e: - if "_DISTUTILS_PATCH" in str(e): - errors.append(str(e)) + _check_distutils_patch(errors) finally: if str(tmp_path) in sys.path: sys.path.remove(str(tmp_path)) diff --git a/tests/integration/test_run_int.py b/tests/integration/test_run_int.py index e1dc6d3f5..243691994 100644 --- a/tests/integration/test_run_int.py +++ b/tests/integration/test_run_int.py @@ -6,22 +6,30 @@ from virtualenv import cli_run from virtualenv.info import IS_PYPY +from virtualenv.seed.wheels.embed import BUNDLE_FOLDER from virtualenv.util.subprocess import run_cmd if TYPE_CHECKING: from pathlib import Path +@pytest.mark.graalpy @pytest.mark.skipif(IS_PYPY, reason="setuptools distutils patching does not work") def test_app_data_pinning(tmp_path: Path) -> None: - version = "23.1" - result = cli_run([str(tmp_path), "--pip", version, "--activators", "", "--seeder", "app-data"]) + # pin a bundled wheel served from a search dir: a version absent locally would be downloaded, and a stalled + # download kills the xdist worker on Windows instead of failing the test + version = "26.0.1" + result = cli_run([ + str(tmp_path), + "--pip", + version, + "--extra-search-dir", + str(BUNDLE_FOLDER), + "--activators", + "", + "--seeder", + "app-data", + ]) code, out, _ = run_cmd([str(result.creator.script("pip")), "list", "--disable-pip-version-check"]) assert not code - for line in out.splitlines(): - parts = line.split() - if parts and parts[0] == "pip": - assert parts[1] == version - break - else: - assert not out + assert ["pip", version] in [line.split() for line in out.splitlines()] diff --git a/tests/integration/test_zipapp.py b/tests/integration/test_zipapp.py index c6c46c7bc..13a6c7703 100644 --- a/tests/integration/test_zipapp.py +++ b/tests/integration/test_zipapp.py @@ -2,12 +2,13 @@ import shutil import subprocess +import zipfile from contextlib import suppress from pathlib import Path import pytest +from python_discovery import PythonInfo -from virtualenv.discovery.py_info import PythonInfo from virtualenv.info import fs_supports_symlink from virtualenv.run import cli_run @@ -18,7 +19,7 @@ @pytest.fixture(scope="session") def zipapp_build_env(tmp_path_factory): create_env_path = None - if CURRENT.implementation not in {"PyPy", "GraalVM"}: + if CURRENT.implementation not in {"PyPy", "GraalPy"}: exe = CURRENT.executable # guaranteed to contain a recent enough pip (tox.ini) else: create_env_path = tmp_path_factory.mktemp("zipapp-create-env") @@ -51,8 +52,14 @@ def zipapp_build_env(tmp_path_factory): else: msg = "could not find a python to build zipapp" raise RuntimeError(msg) - cmd = [str(Path(exe).parent / "pip"), "install", "pip>=23", "packaging>=23"] - subprocess.check_call(cmd) + cmd = [ + str(Path(exe).parent / "pip"), + "install", + "pip>=23", + "packaging>=23", + "tomli>=2.0.1; python_version<'3.11'", + ] + subprocess.run(cmd, check=True, timeout=300) yield exe if create_env_path is not None: shutil.rmtree(str(create_env_path)) @@ -64,7 +71,7 @@ def zipapp(zipapp_build_env, tmp_path_factory): path = HERE.parent.parent / "tasks" / "make_zipapp.py" filename = into / "virtualenv.pyz" cmd = [zipapp_build_env, str(path), "--dest", str(filename)] - subprocess.check_call(cmd) + subprocess.run(cmd, check=True, timeout=300) yield filename shutil.rmtree(str(into)) @@ -78,39 +85,56 @@ def zipapp_test_env(tmp_path_factory): @pytest.fixture -def call_zipapp(zipapp, tmp_path, zipapp_test_env, temp_app_data): # noqa: ARG001 - def _run(*args): +def call_zipapp(zipapp, tmp_path, zipapp_test_env, temp_app_data): # ruff:ignore[unused-function-argument] + def _run(*args) -> None: cmd = [str(zipapp_test_env), str(zipapp), "-vv", str(tmp_path / "env"), *list(args)] - subprocess.check_call(cmd) + subprocess.run(cmd, check=True, timeout=120) return _run @pytest.fixture -def call_zipapp_symlink(zipapp, tmp_path, zipapp_test_env, temp_app_data): # noqa: ARG001 - def _run(*args): +def call_zipapp_symlink(zipapp, tmp_path, zipapp_test_env, temp_app_data): # ruff:ignore[unused-function-argument] + def _run(*args) -> None: symlinked = zipapp.parent / "symlinked_virtualenv.pyz" symlinked.symlink_to(str(zipapp)) cmd = [str(zipapp_test_env), str(symlinked), "-vv", str(tmp_path / "env"), *list(args)] - subprocess.check_call(cmd) + subprocess.run(cmd, check=True, timeout=120) return _run +@pytest.mark.timeout(600) @pytest.mark.skipif(not fs_supports_symlink(), reason="symlink not supported") -def test_zipapp_in_symlink(capsys, call_zipapp_symlink): +def test_zipapp_in_symlink(capsys, call_zipapp_symlink) -> None: call_zipapp_symlink("--reset-app-data") _out, err = capsys.readouterr() assert not err -def test_zipapp_help(call_zipapp, capsys): - call_zipapp("-h") - _out, err = capsys.readouterr() - assert not err - - +@pytest.mark.timeout(600) @pytest.mark.slow @pytest.mark.parametrize("seeder", ["app-data", "pip"]) -def test_zipapp_create(call_zipapp, seeder): +def test_zipapp_create(call_zipapp, seeder) -> None: call_zipapp("--seeder", seeder) + + +@pytest.mark.timeout(600) +def test_zipapp_bundles_prebuilt_wheel(zipapp_build_env: str, tmp_path: Path) -> None: + (wheel_dir := tmp_path / "wheels").mkdir() + with zipfile.ZipFile(wheel_dir / "virtualenv-0-py3-none-any.whl", "w") as wheel: + wheel.writestr("virtualenv/prebuilt.txt", "prebuilt") + subprocess.run( + [ + zipapp_build_env, + str(HERE.parent.parent / "tasks" / "make_zipapp.py"), + "--dest", + str(pyz := tmp_path / "virtualenv.pyz"), + "--wheel-dir", + str(wheel_dir), + ], + check=True, + timeout=300, + ) + with zipfile.ZipFile(pyz) as archive: + assert archive.read("virtualenv/prebuilt.txt") == b"prebuilt" diff --git a/tests/property/conftest.py b/tests/property/conftest.py new file mode 100644 index 000000000..b5e7a985e --- /dev/null +++ b/tests/property/conftest.py @@ -0,0 +1,12 @@ +"""Keep the property tests out of collection when hypothesis is absent. + +The marker deselection in addopts runs after import, so without this the default test environments, which do not install +hypothesis, fail at collection rather than skipping. + +""" + +from __future__ import annotations + +from importlib.util import find_spec + +collect_ignore_glob = [] if find_spec("hypothesis") else ["*.py"] diff --git a/tests/property/test_batch_quote.py b/tests/property/test_batch_quote.py new file mode 100644 index 000000000..faa2b876d --- /dev/null +++ b/tests/property/test_batch_quote.py @@ -0,0 +1,72 @@ +r"""Property tests for BatchActivator.quote(). + +``@set "VAR=value"`` is the only place a value from ``--prompt`` (or any other replacement) reaches cmd.exe, and cmd.exe +treats several characters as live syntax there regardless of the surrounding quotes - confirmed on a real Windows runner +while fixing the ``activate.bat`` injection this closes. The invariant that matters is that none of those characters can +survive ``quote()``. + +``(`` and ``)`` are deliberately not in that set: also confirmed on a real Windows runner, they round-trip unchanged +through the same construct, since cmd.exe only treats them as special when they delimit a control-flow block (``if``, +``for``), not as bare characters in a value. Neutering them anyway would corrupt a destination path under the extremely +common ``C:\Program Files (x86)\...``. + +""" + +from __future__ import annotations + +import pytest +from hypothesis import example, given +from hypothesis import strategies as st + +from virtualenv.activation.batch import BatchActivator + +pytestmark = pytest.mark.property + +# Confirmed live syntax inside @set "VAR=value" on a real Windows runner, independently of this fix's +# own implementation - this is not just re-asserting quote()'s source. `!` only turns live when the +# caller already has delayed expansion enabled, but a crafted prompt then leaks the value of whatever +# variable it names, confirmed the same way. +DANGEROUS = '"&|<>^!' +SAFE_IN_THIS_CONTEXT = "()" +LINE_BOUNDARIES = "\n\r\v\f\x1c\x1d\x1e\x85" + chr(0x2028) + chr(0x2029) + +# Lone surrogates are excluded: cmd.exe scripts are read as text, and the same limit applies to the +# pyvenv.cfg property tests for the same reason. +values = st.text(st.characters(blacklist_categories=("Cs",), max_codepoint=0x2100)) + + +@given(value=values) +@example('x" & type nul > pwned.marker & "') # the confirmed exploit payload +@example("x^& type nul ^> pwned.marker ^& ") # caret-escaped: still live outside a quoted context +@example("x\n@type nul > pwned.marker\n@rem ") # newline: a full extra statement, not just a breakout +@example('a"&type nul>pwned.marker&"b') +@example("foo!SECRET!bar") # delayed-expansion variable substitution, confirmed under cmd /V:ON +def test_quote_strips_every_dangerous_character(value: str) -> None: + quoted = BatchActivator.quote(value) + + assert not (set(quoted) & set(DANGEROUS)) + assert not (set(quoted) & set(LINE_BOUNDARIES)) + + +@given(value=st.text(st.sampled_from(SAFE_IN_THIS_CONTEXT), min_size=1)) +@example("(x86)") # the shape that shows up in every 32-bit "Program Files (x86)" install +def test_quote_leaves_parentheses_untouched(value: str) -> None: + assert BatchActivator.quote(value) == value + + +@given(value=values) +def test_quote_keeps_percent_literal_and_balanced(value: str) -> None: + """Every ``%`` must survive doubled, never as a lone character that triggers expansion.""" + quoted = BatchActivator.quote(value) + + assert quoted.replace("%%", "").count("%") == 0 + + +@given(value=values) +@example('x" & type nul > pwned.marker & "') +def test_embedding_in_the_set_statement_stays_one_well_formed_line(value: str) -> None: + """The exact shape the template uses: a single ``@set "VAR=value"`` statement.""" + rendered = f'@set "VIRTUAL_ENV_PROMPT={BatchActivator.quote(value)}"' + + assert rendered.count("\n") == 0 + assert rendered.count('"') == 2 diff --git a/tests/property/test_pyenv_cfg.py b/tests/property/test_pyenv_cfg.py new file mode 100644 index 000000000..673ad3129 --- /dev/null +++ b/tests/property/test_pyenv_cfg.py @@ -0,0 +1,77 @@ +"""Property tests for the pyvenv.cfg reader and writer. + +pyvenv.cfg is a line-based format with no escape syntax, so the invariant that matters is that a value can never be read +back as extra configuration. These properties are what caught #3247: a prompt containing a line boundary used to inject +arbitrary keys, overriding any key written before it, including ``home``. + +""" + +from __future__ import annotations + +import os +from collections import OrderedDict + +import pytest +from hypothesis import example, given +from hypothesis import strategies as st + +from virtualenv.create.pyenv_cfg import PyEnvCfg + +pytestmark = pytest.mark.property + +# Anything str.splitlines() treats as a boundary is what makes a value escape its own line. +LINE_BOUNDARIES = "\n\r\v\f\x1c\x1d\x1e\x85" + chr(0x2028) + chr(0x2029) + +# Lone surrogates are excluded throughout: the file is UTF-8, so they cannot be written at all, and +# the same is true of the stdlib venv. That is a limit of the format rather than of this code. +printable = st.characters(blacklist_categories=("Cs",), max_codepoint=0x2100) + +keys = st.text( + st.characters(blacklist_categories=("Cs",), blacklist_characters="=" + LINE_BOUNDARIES), min_size=1 +).filter(lambda k: k.strip() == k and not k.startswith("#")) +contents = st.dictionaries(keys, st.text(printable), max_size=6) + + +def _round_trip(content: dict[str, str], tmp_path) -> dict[str, str]: + path = tmp_path / "pyvenv.cfg" + PyEnvCfg(OrderedDict(content), path).write() + return dict(PyEnvCfg.from_file(path).content) + + +@given(content=contents) +@example(content={"prompt": 'x"\nhome = /attacker\nprompt = "z'}) +@example(content={"prompt": "a" + chr(0x2028) + "home = /attacker"}) +@example(content={"home": "a\r\nevil = 1"}) +def test_writing_then_reading_never_invents_keys(content, tmp_path_factory) -> None: + """A value must never be read back as a new configuration key.""" + read_back = _round_trip(content, tmp_path_factory.mktemp("cfg")) + + assert set(read_back) <= set(content) + + +@given(content=contents) +@example(content={"prompt": 'x"\nhome = /attacker\nprompt = "z'}) +def test_written_file_has_one_line_per_key(content, tmp_path_factory) -> None: + """However exotic the values, the file keeps its line-per-key shape.""" + path = tmp_path_factory.mktemp("cfg") / "pyvenv.cfg" + PyEnvCfg(OrderedDict(content), path).write() + + lines = path.read_text(encoding="utf-8").splitlines() + + assert len(lines) == len(content) + assert all(" = " in line for line in lines) + + +@given( + content=st.dictionaries( + keys, + st.text(st.characters(blacklist_categories=("Cs",), blacklist_characters=LINE_BOUNDARIES), min_size=1).filter( + # write() rewrites a value naming an existing path, which is deliberate and not under test. + lambda v: v.strip() == v and v[0] not in "'\"" and v[-1] not in "'\"" and not os.path.exists(v) + ), + max_size=6, + ) +) +def test_plain_values_round_trip_unchanged(content, tmp_path_factory) -> None: + """A value that the format can represent comes back exactly as written.""" + assert _round_trip(content, tmp_path_factory.mktemp("cfg")) == content diff --git a/tests/tasks/test_graalpy_runner.py b/tests/tasks/test_graalpy_runner.py new file mode 100644 index 000000000..ab511ab98 --- /dev/null +++ b/tests/tasks/test_graalpy_runner.py @@ -0,0 +1,205 @@ +from __future__ import annotations + +import io +import os +import runpy +import subprocess +import sys +from pathlib import Path +from typing import TYPE_CHECKING, Final +from unittest.mock import create_autospec + +import pytest + +if TYPE_CHECKING: + from collections.abc import Callable, Sequence + from unittest.mock import MagicMock + + from pytest_mock import MockerFixture + +_CLEAN: Final[str] = '' +_EMPTY: Final[str] = '' +_SHUTDOWN: Final[str] = "did not complete all polyglot threads" +_PASS: Final[tuple[int, str, str | None]] = (0, "", _CLEAN) +_POPEN: Final[type[subprocess.Popen[str]]] = subprocess.Popen # the fixture patches the module attribute +_RUNNER: Final[Path] = Path(__file__).parents[2] / "tasks" / "run_graalpy_tests.py" + +Outcome = tuple[int, str, "str | None"] + + +@pytest.fixture +def junit(tmp_path: Path) -> Path: + return tmp_path / "junit.xml" + + +@pytest.fixture +def popen(mocker: MockerFixture, junit: Path) -> Callable[[Sequence[Outcome]], MagicMock]: + def install(outcomes: Sequence[Outcome]) -> MagicMock: + for index in range(1, len(outcomes) + 1): + junit.with_suffix(f".{index}.xml").write_text(_CLEAN, encoding="utf-8") # a stale report from a prior run + + def start(command: list[str], **_kwargs: object) -> MagicMock: + index = int(command[-3].removeprefix("--shard=").split("/")[0]) + status, stderr, report = outcomes[index - 1] + if report is not None: + Path(command[-1]).write_text(report, encoding="utf-8") + process = create_autospec( + _POPEN, instance=True, stdout=io.StringIO(f"collected {index}\n"), stderr=io.StringIO(stderr) + ) + # Python 3.11 autospec replaces a child configured through create_autospec kwargs + process.wait.return_value = status + return process + + return mocker.patch("subprocess.Popen", autospec=True, side_effect=start) + + return install + + +@pytest.fixture +def run_shards( + mocker: MockerFixture, junit: Path, popen: Callable[[Sequence[Outcome]], MagicMock] +) -> Callable[[Sequence[Outcome]], MagicMock]: + def invoke(outcomes: Sequence[Outcome]) -> MagicMock: + started = popen(outcomes) + mocker.patch.object(sys, "argv", ["runner", "--shards", str(len(outcomes)), str(junit), "tests"]) + runpy.run_path(str(_RUNNER), run_name="__main__") + return started + + return invoke + + +@pytest.mark.parametrize( + "outcomes", + [ + pytest.param([_PASS, _PASS], id="clean"), + pytest.param([(1, _SHUTDOWN, _CLEAN), _PASS], id="known-shutdown"), + pytest.param([(1, _SHUTDOWN, f"{_CLEAN}"), _PASS], id="wrapped-suite"), + pytest.param([(5, "", _EMPTY), _PASS], id="shard-without-tests"), + pytest.param([(1, _SHUTDOWN, _EMPTY), _PASS], id="shutdown-without-tests"), + ], +) +def test_graalpy_shards_pass(run_shards: Callable[[Sequence[Outcome]], MagicMock], outcomes: list[Outcome]) -> None: + run_shards(outcomes) + + +@pytest.mark.parametrize( + "first", + [ + pytest.param((1, "unrelated crash", _CLEAN), id="unknown-crash"), + pytest.param((2, _SHUTDOWN, _CLEAN), id="interrupted"), + pytest.param((3, _SHUTDOWN, _CLEAN), id="internal-error"), + pytest.param((4, _SHUTDOWN, _CLEAN), id="usage-error"), + pytest.param((-9, _SHUTDOWN, _CLEAN), id="killed"), + pytest.param((0, "", None), id="success-without-report"), + pytest.param((1, _SHUTDOWN, None), id="stale-report"), + pytest.param((1, _SHUTDOWN, ""), id="no-suites"), + pytest.param((1, _SHUTDOWN, _CLEAN.replace('failures="0"', 'failures="1"')), id="failed-test"), + pytest.param((1, _SHUTDOWN, _CLEAN.replace('errors="0"', 'errors="1"')), id="errored-test"), + pytest.param((1, _SHUTDOWN, _CLEAN.replace('tests="1"', "")), id="missing-count"), + pytest.param((1, _SHUTDOWN, _CLEAN.replace('tests="1"', 'tests="bad"')), id="invalid-count"), + pytest.param( + (1, _SHUTDOWN, f'{_CLEAN}'), + id="later-suite-fails", + ), + ], +) +def test_graalpy_shard_fails(run_shards: Callable[[Sequence[Outcome]], MagicMock], first: Outcome) -> None: + with pytest.raises(SystemExit, match=r"^pytest shards failed: 1/2$"): + run_shards([first, _PASS]) + + +def test_graalpy_shards_report_every_failure(run_shards: Callable[[Sequence[Outcome]], MagicMock]) -> None: + with pytest.raises(SystemExit, match=r"^pytest shards failed: 1/2, 2/2$"): + run_shards([(2, "", _CLEAN), (2, "", _CLEAN)]) + + +@pytest.mark.parametrize( + "report", + [ + pytest.param(_EMPTY, id="zero-tests"), + pytest.param(_CLEAN.replace('skipped="0"', 'skipped="1"'), id="all-skipped"), + ], +) +def test_graalpy_shards_fail_without_tests(run_shards: Callable[[Sequence[Outcome]], MagicMock], report: str) -> None: + with pytest.raises(SystemExit, match=r"^pytest shards ran no tests$"): + run_shards([(5, "", report), (0, "", report)]) + + +def test_graalpy_shard_command(run_shards: Callable[[Sequence[Outcome]], MagicMock], junit: Path) -> None: + started = run_shards([_PASS, _PASS]) + assert [call.args[0] for call in started.call_args_list] == [ + [sys.executable, "-u", "-m", "pytest", "tests", f"--shard={index}/2", "--junitxml", str(junit.parent / name)] + for index, name in ((1, "junit.1.xml"), (2, "junit.2.xml")) + ] + + +def test_graalpy_shard_output_prefixed( + run_shards: Callable[[Sequence[Outcome]], MagicMock], capsys: pytest.CaptureFixture[str] +) -> None: + run_shards([(1, f"{_SHUTDOWN}\n", _CLEAN), _PASS]) + assert capsys.readouterr() == ( + ( + "[1/2] collected 1\n" + "[2/2] collected 2\n" + "[1/2] GraalVM reported its known polyglot shutdown error after a completed test run (see #3240).\n" + ), + f"[1/2] {_SHUTDOWN}\n", + ) + + +@pytest.mark.parametrize( + ("status", "report", "message"), + [ + pytest.param(2, _CLEAN, "[1/2] exit 2\n", id="exit-status"), + pytest.param(0, None, "[1/2] exit 0 without a clean junit report\n", id="bad-report"), + ], +) +def test_graalpy_shard_failure_explained( + run_shards: Callable[[Sequence[Outcome]], MagicMock], + capsys: pytest.CaptureFixture[str], + status: int, + report: str | None, + message: str, +) -> None: + with pytest.raises(SystemExit): + run_shards([(status, "", report), _PASS]) + assert capsys.readouterr().out.endswith(message) + + +@pytest.mark.parametrize( + ("cpu_count", "expected"), + [pytest.param(3, 3, id="per-cpu"), pytest.param(None, 1, id="unknown-cpu-count")], +) +def test_graalpy_shards_default_to_cpu_count( + mocker: MockerFixture, + junit: Path, + popen: Callable[[Sequence[Outcome]], MagicMock], + cpu_count: int | None, + expected: int, +) -> None: + mocker.patch.object(os, "cpu_count", return_value=cpu_count) + started = popen([_PASS] * expected) + mocker.patch.object(sys, "argv", ["runner", str(junit), "tests"]) + runpy.run_path(str(_RUNNER), run_name="__main__") + assert started.call_count == expected + + +def test_graalpy_groups_cover_all_shards( + mocker: MockerFixture, junit: Path, popen: Callable[[Sequence[Outcome]], MagicMock] +) -> None: + started: Final[MagicMock] = popen([_PASS] * 8) + for group in range(1, 5): + mocker.patch.object(sys, "argv", ["runner", "--shards", "2", "--group", f"{group}/4", str(junit), "tests"]) + runpy.run_path(str(_RUNNER), run_name="__main__") + assert sorted(call.args[0][-3] for call in started.call_args_list) == [ + f"--shard={index}/8" for index in range(1, 9) + ] + + +@pytest.mark.parametrize("group", ["0/4", "5/4", "1/0", "1", "a/4", "1/2/3"], ids=str) +def test_graalpy_invalid_group(mocker: MockerFixture, junit: Path, group: str) -> None: + mocker.patch.object(sys, "argv", ["runner", "--group", group, str(junit), "tests"]) + with pytest.raises(SystemExit) as exc: + runpy.run_path(str(_RUNNER), run_name="__main__") + assert exc.value.code == 2 diff --git a/tests/tasks/test_pytest_diagnostics.py b/tests/tasks/test_pytest_diagnostics.py new file mode 100644 index 000000000..20f7b25b8 --- /dev/null +++ b/tests/tasks/test_pytest_diagnostics.py @@ -0,0 +1,208 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Final + +import pytest + +pytest_plugins = ["pytester"] + +_WAIT: Final[str] = """ +import os +import time +from pathlib import Path +from typing import Final + +def wait_for_stack(function: str) -> None: + worker: Final[str] = os.environ.get("PYTEST_XDIST_WORKER", "controller") + path: Final[Path] = Path("diagnostics") / f"{worker}-{os.getpid()}.log" + deadline: Final[float] = time.monotonic() + 20 + while time.monotonic() < deadline: + text = path.read_text(encoding="utf-8") + controller = next(Path("diagnostics").glob("controller-*.log")).read_text(encoding="utf-8") + if ( + text.count(" DUMP\\n") >= 2 + and f"in {function}\\n" in text + and all("pytest_runtestloop" in log for log in (text, controller)) + ): + return + time.sleep(0.01) + raise TimeoutError("stack dumps did not capture the blocked pytest process") +""" + + +@pytest.fixture +def diagnostic_pytester(pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch) -> pytest.Pytester: + monkeypatch.setenv("PYTHONPATH", str(Path(__file__).parents[2])) + monkeypatch.setenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", "1") + monkeypatch.setenv("PYTEST_ADDOPTS", "") + monkeypatch.delenv("PYTEST_XDIST_WORKER", raising=False) + pytester.plugins.append("tasks.pytest_diagnostics") + pytester.makeini("[pytest]\naddopts = --diagnostics-dir=diagnostics --diagnostics-interval=0.05\n") + return pytester + + +def test_diagnostics_capture_controller_and_workers(diagnostic_pytester: pytest.Pytester) -> None: + source: Final[str] = _WAIT + '\ndef test_wait_for_stack() -> None:\n wait_for_stack("test_wait_for_stack")\n' + diagnostic_pytester.makepyfile(test_first=source, test_second=source) + diagnostic_pytester.runpytest_subprocess("-p", "xdist.plugin", "-n", "2", timeout=30).assert_outcomes(passed=2) + logs: Final[dict[str, str]] = { + path.name.split("-")[0]: path.read_text(encoding="utf-8") + for path in (diagnostic_pytester.path / "diagnostics").glob("*.log") + } + assert set(logs) == {"controller", "gw0", "gw1"} + for worker, text in logs.items(): + assert text.count(" DUMP\n") >= 2 + assert "pytest_runtestloop" in text + assert "pid=" in text + assert "parent=" in text + assert "python=" in text + assert "TEST test_" in text + assert "call passed test_" in text + assert text.endswith("EXIT\n") + if worker != "controller": + assert "in test_wait_for_stack\n" in text + + +def test_diagnostics_rearm_after_failure(diagnostic_pytester: pytest.Pytester) -> None: + diagnostic_pytester.makepyfile( + _WAIT + + """ +def test_failure() -> None: + raise RuntimeError("intentional failure") + +def test_wait_for_stack() -> None: + wait_for_stack("test_wait_for_stack") +""" + ) + result: Final[pytest.RunResult] = diagnostic_pytester.runpytest_subprocess(timeout=30) + result.assert_outcomes(failed=1, passed=1) + text: Final[str] = next((diagnostic_pytester.path / "diagnostics").glob("*.log")).read_text(encoding="utf-8") + assert "call failed" in text + assert "in test_wait_for_stack\n" in text + assert " DUMP\n" in text + + +def test_diagnostics_survive_abrupt_exit(diagnostic_pytester: pytest.Pytester) -> None: + diagnostic_pytester.makepyfile( + _WAIT + + """ +def test_wait_for_stack() -> None: + wait_for_stack("test_wait_for_stack") + os._exit(23) +""" + ) + assert diagnostic_pytester.runpytest_subprocess(timeout=30).ret == 23 + text: Final[str] = next((diagnostic_pytester.path / "diagnostics").glob("*.log")).read_text(encoding="utf-8") + assert "in test_wait_for_stack\n" in text + assert " DUMP\n" in text + assert "STOP" not in text + + +@pytest.mark.parametrize("nested", [False, True], ids=["direct", "after-nested-pytest"]) +def test_diagnostics_retain_worker_crash(diagnostic_pytester: pytest.Pytester, nested: bool) -> None: + diagnostic_pytester.makepyfile( + inner="def test_inner() -> None: pass", + outer=f""" +import os +import pytest + +def test_crash() -> None: + if {nested}: + assert pytest.main(["inner.py", "-p", "tasks.pytest_diagnostics"]) == 0 + os.abort() +""", + ) + # the watchdog and the fatal handler write to the same file unsynchronized, so keep the watchdog quiet here + diagnostic_pytester.runpytest_subprocess( + "outer.py", "-p", "xdist.plugin", "-n", "1", "--max-worker-restart=0", "--diagnostics-interval=60", timeout=30 + ).assert_outcomes(failed=1) + text: Final[str] = next((diagnostic_pytester.path / "diagnostics").glob("gw0-*.log")).read_text(encoding="utf-8") + assert "Fatal Python error:" in text + assert "in test_crash\n" in text + + +def test_diagnostics_survive_nested_pytest(diagnostic_pytester: pytest.Pytester) -> None: + diagnostic_pytester.makepyfile( + inner="def test_inner() -> None: pass", + outer=_WAIT + + """ +import pytest + +def test_wait_after_nested_pytest() -> None: + assert pytest.main(["inner.py", "-p", "tasks.pytest_diagnostics", "--diagnostics-dir=nested"]) == 0 + wait_for_stack("test_wait_after_nested_pytest") +""", + ) + diagnostic_pytester.runpytest_subprocess("outer.py", timeout=30).assert_outcomes(passed=1) + text: Final[str] = next((diagnostic_pytester.path / "diagnostics").glob("*.log")).read_text(encoding="utf-8") + assert "in test_wait_after_nested_pytest\n" in text + + +def test_diagnostics_dump_when_gil_held(diagnostic_pytester: pytest.Pytester) -> None: + # a backtracking regex keeps the GIL for over a second, longer than the stall timer's 10 intervals + diagnostic_pytester.makepyfile( + """ +import re +import time +from pathlib import Path + +def test_hold_the_gil() -> None: + log = next(Path("diagnostics").glob("controller-*.log")) + deadline = time.monotonic() + 20 + while "Timeout (" not in log.read_text(encoding="utf-8"): + assert time.monotonic() < deadline, "the stall timer never fired" + re.fullmatch(r"(a+)+b", "a" * 26) +""" + ) + diagnostic_pytester.runpytest_subprocess(timeout=30).assert_outcomes(passed=1) + text: Final[str] = next((diagnostic_pytester.path / "diagnostics").glob("*.log")).read_text(encoding="utf-8") + _, header, after = text.partition("Timeout (") + assert (header, "in test_hold_the_gil\n" in after.split(" DUMP\n", 1)[0]) == ("Timeout (", True) + + +def test_diagnostics_capture_shutdown_wait(diagnostic_pytester: pytest.Pytester) -> None: + diagnostic_pytester.makeconftest( + """ +import os +import threading +import time +from pathlib import Path +from typing import Final + +def pytest_unconfigure() -> None: + threading.Thread(target=wait_for_shutdown).start() + +def wait_for_shutdown() -> None: + path: Final[Path] = Path("diagnostics") / f"controller-{os.getpid()}.log" + deadline: Final[float] = time.monotonic() + 20 + while time.monotonic() < deadline: + tail = path.read_text(encoding="utf-8").rsplit("STOP\\n", 1)[-1] + if tail.count("in _shutdown\\n") >= 2 and "in wait_for_shutdown\\n" in tail: + return + time.sleep(0.01) + raise TimeoutError("no stack dump during shutdown") +""" + ) + diagnostic_pytester.makepyfile("def test_empty() -> None: pass") + diagnostic_pytester.runpytest_subprocess(timeout=30).assert_outcomes(passed=1) + text: Final[str] = next((diagnostic_pytester.path / "diagnostics").glob("*.log")).read_text(encoding="utf-8") + tail: Final[str] = text.rsplit("STOP\n", 1)[-1] + assert tail.count("in _shutdown\n") >= 2 + assert "in wait_for_shutdown\n" in tail + + +def test_diagnostics_default_directory(diagnostic_pytester: pytest.Pytester) -> None: + diagnostic_pytester.makeini("[pytest]\n") + diagnostic_pytester.makepyfile("def test_empty() -> None: pass") + diagnostic_pytester.runpytest_subprocess(timeout=30).assert_outcomes(passed=1) + text: Final[str] = next((diagnostic_pytester.path / ".tox/hang-diagnostics").glob("*.log")).read_text( + encoding="utf-8" + ) + assert "call passed test_" in text + + +def test_diagnostics_reject_nonpositive_interval(diagnostic_pytester: pytest.Pytester) -> None: + result: Final[pytest.RunResult] = diagnostic_pytester.runpytest_subprocess("--diagnostics-interval=0", timeout=30) + assert result.ret == pytest.ExitCode.USAGE_ERROR + result.stderr.fnmatch_lines(["*--diagnostics-interval must be positive*"]) diff --git a/tests/tasks/test_release.py b/tests/tasks/test_release.py new file mode 100644 index 000000000..5e1a8c4df --- /dev/null +++ b/tests/tasks/test_release.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +import os +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Final + +from git import Repo + + +def test_release_without_tools_on_path(tmp_path: Path) -> None: + remote: Final[Repo] = Repo.init(tmp_path / "pypa" / "virtualenv", bare=True) + repo: Final[Repo] = Repo.init(tmp_path / "checkout") + root: Final[Path] = Path(repo.working_dir) + repo.create_remote("origin", str(remote.git_dir)) + with repo.config_writer() as config: + config.set_value("user", "name", "Test") + config.set_value("user", "email", "test@example.com") + (root / "tasks").mkdir() + shutil.copyfile(Path(__file__).parents[2] / "tasks" / "release.py", root / "tasks" / "release.py") + (root / "docs" / "changelog").mkdir(parents=True) + fragment: Final[str] = "The release tools use the active interpreter. " * 6 + (root / "docs" / "changelog" / "1.bugfix.rst").write_text(fragment + "\n", encoding="utf-8") + (root / "pyproject.toml").write_text( + '[tool.towncrier]\nname = "demo"\ndirectory = "docs/changelog"\nfilename = "docs/changelog.rst"\n', + encoding="utf-8", + ) + repo.git.add(".") + repo.index.commit("initial") + git: Final[str | None] = shutil.which("git") + assert git is not None + command_dir: Final[Path] = tmp_path / "bin" + command_dir.mkdir() + (command_dir / Path(git).name).symlink_to(git) + result: Final[subprocess.CompletedProcess[str]] = subprocess.run( + [sys.executable, "tasks/release.py", "--version", "1.2.3", "--no-push"], + cwd=root, + env={**os.environ, "PATH": str(command_dir), "GIT_PYTHON_GIT_EXECUTABLE": git}, + text=True, + capture_output=True, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + changelog: Final[str] = (root / "docs" / "changelog.rst").read_text(encoding="utf-8") + assert fragment.strip() in " ".join(changelog.split()) + assert max(map(len, changelog.splitlines())) <= 120 + assert repo.tags["1.2.3"].commit == repo.head.commit + assert repo.head.commit.message == "release 1.2.3" + assert not repo.is_dirty(untracked_files=True) + assert not remote.tags diff --git a/tests/tasks/test_sbom.py b/tests/tasks/test_sbom.py new file mode 100644 index 000000000..5e525f041 --- /dev/null +++ b/tests/tasks/test_sbom.py @@ -0,0 +1,721 @@ +from __future__ import annotations + +import base64 +import csv +import hashlib +import json +import runpy +import shutil +import sys +import zipfile +from io import BytesIO, StringIO +from pathlib import Path +from typing import TYPE_CHECKING, Any, Final + +import pytest +from hatchling.builders.wheel import WheelBuilder + +if TYPE_CHECKING: + from collections.abc import Callable + + from pytest_mock import MockerFixture + +_DISTRIBUTIONS: Final[str] = json.dumps({ + "3.14": {"==any": {"distlib": "__virtualenv__/distlib-0.4-py3-none-any/distlib-0.4.dist-info"}}, + "3.9": {"==any": {"distlib": "__virtualenv__/distlib-0.4-py3-none-any/distlib-0.4.dist-info"}}, +}) + +_ALLOWED: Final[str] = "['Apache-2.0', 'BSD-2-Clause', 'BSD-3-Clause', 'ISC', 'MIT', 'PSF-2.0', 'Unlicense']" + + +def _expression(expression: str) -> dict[str, Any]: + return {"expression": expression, "acknowledgement": "declared"} + + +def _named(name: str) -> dict[str, Any]: + return {"license": {"name": name, "acknowledgement": "declared"}} + + +def _concluded(identifier: str) -> dict[str, Any]: + return {"license": {"id": identifier, "acknowledgement": "concluded"}} + + +def _denied(expression: str, bom_ref: str = "pkg:pypi/example@1.0") -> str: + return f"{bom_ref}: license {expression} is outside ALLOWED_LICENSES {_ALLOWED}" + + +@pytest.mark.parametrize( + "filename", + [ + pytest.param("pip/example.py", id="plain"), + pytest.param("pip/a,b.py", id="comma"), + pytest.param('pip/a"b.py', id="quote"), + pytest.param("pip/a\nb.py", id="newline"), + ], +) +def test_sbom_record_paths(build_sbom: Callable[[str, dict[str, str]], str], filename: str) -> None: + assert json.loads(build_sbom(filename, {}))["components"][0]["components"] == [ + { + "type": "file", + "bom-ref": f"pkg:pypi/pip@1.0#{filename}", + "name": filename, + "hashes": [ + {"alg": "SHA-256", "content": "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"} + ], + "properties": [{"name": "size", "value": "3"}], + }, + ] + + +def test_sbom_unresolved_dependencies(build_sbom: Callable[[str, dict[str, str]], str]) -> None: + document: Final[dict[str, Any]] = json.loads(build_sbom("pip/example.py", {})) + assert [entry for entry in document["dependencies"] if not entry["ref"].startswith("tool:")] == [ + {"ref": "pkg:pypi/virtualenv@1.0", "dependsOn": ["pkg:pypi/pip@1.0", "requires-dist:python-discovery>=1.6"]}, + ] + + +@pytest.mark.parametrize( + ("source", "content"), + [ + pytest.param( + "pip/_vendor/vendor.txt", "# bundled\n\n urllib3==1.26.4 # patched upstream\n", id="pip-manifest" + ), + pytest.param( + "pip/_vendor/urllib3-1.26.4.dist-info/METADATA", "Name: urllib3\nVersion: 1.26.4\n", id="nested-metadata" + ), + ], +) +def test_sbom_vendored_identity(build_sbom: Callable[[str, dict[str, str]], str], source: str, content: str) -> None: + document: Final[dict[str, Any]] = json.loads(build_sbom("pip/example.py", {source: content})) + assert [child for child in document["components"][0]["components"] if child["type"] == "library"] == [ + { + "type": "library", + "bom-ref": "pkg:pypi/pip@1.0#vendored/pkg:pypi/urllib3@1.26.4", + "name": "urllib3", + "version": "1.26.4", + "purl": "pkg:pypi/urllib3@1.26.4", + "externalReferences": [], + "properties": [{"name": "virtualenv:vendored-manifest", "value": source}], + "evidence": { + "identity": [ + { + "field": "purl", + "confidence": 1, + "methods": [{"technique": "manifest-analysis", "confidence": 1, "value": source}], + } + ] + }, + } + ] + + +def test_sbom_vendored_relationship(build_sbom: Callable[[str, dict[str, str]], str]) -> None: + document: Final[dict[str, Any]] = json.loads( + build_sbom("pip/example.py", {"pip/_vendor/vendor.txt": "urllib3==1.26.4\n"}) + ) + assert [entry for entry in document["dependencies"] if not entry["ref"].startswith("tool:")] == [ + {"ref": "pkg:pypi/virtualenv@1.0", "dependsOn": ["pkg:pypi/pip@1.0", "requires-dist:python-discovery>=1.6"]}, + {"ref": "pkg:pypi/pip@1.0", "dependsOn": ["pkg:pypi/pip@1.0#vendored/pkg:pypi/urllib3@1.26.4"]}, + ] + + +def test_sbom_vendored_metadata_precedence(build_sbom: Callable[[str, dict[str, str]], str]) -> None: + document: Final[dict[str, Any]] = json.loads( + build_sbom( + "pip/example.py", + { + "pip/_vendor/vendor.txt": "urllib3==1.26.4\n", + "pip/_vendor/urllib3-1.26.4.dist-info/METADATA": "Name: urllib3\nVersion: 1.26.4\nLicense-Expression: MIT\n", + }, + ) + ) + assert [child["licenses"] for child in document["components"][0]["components"] if child["type"] == "library"] == [ + [{"expression": "MIT", "acknowledgement": "declared"}], + ] + + +@pytest.mark.parametrize( + ("metadata", "expected"), + [ + pytest.param( + "Classifier: License :: OSI Approved :: MIT License\n", + [ + {"license": {"name": "MIT License", "acknowledgement": "declared"}}, + {"license": {"id": "MIT", "acknowledgement": "concluded"}}, + ], + id="classifier", + ), + pytest.param( + "Classifier: License :: OSI Approved :: Python Software Foundation License\nLicense: PSF-2.0\n", + [ + {"license": {"name": "Python Software Foundation License", "acknowledgement": "declared"}}, + {"license": {"name": "PSF-2.0", "acknowledgement": "declared"}}, + {"license": {"id": "PSF-2.0", "acknowledgement": "concluded"}}, + ], + id="classifier-and-id-agree", + ), + pytest.param( + "License: mit\n", + [ + {"license": {"name": "mit", "acknowledgement": "declared"}}, + {"license": {"id": "MIT", "acknowledgement": "concluded"}}, + ], + id="id-canonical-case", + ), + pytest.param( + "Classifier: License :: OSI Approved :: MIT License\n" + "Classifier: License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)\n", + [ + {"license": {"name": "MIT License", "acknowledgement": "declared"}}, + {"license": {"name": "GNU Lesser General Public License v3 (LGPLv3)", "acknowledgement": "declared"}}, + ], + id="one-unresolved", + ), + pytest.param( + "License: MIT OR Apache-2.0\n", + [{"license": {"name": "MIT OR Apache-2.0", "acknowledgement": "declared"}}], + id="expression-in-license-field", + ), + pytest.param( + "License: LicenseRef-Proprietary\n", + [{"license": {"name": "LicenseRef-Proprietary", "acknowledgement": "declared"}}], + id="license-ref", + ), + ], +) +def test_sbom_license_resolution( + build_sbom: Callable[[str, dict[str, str]], str], metadata: str, expected: list[dict[str, Any]] +) -> None: + vendored: Final[dict[str, str]] = { + "pip/_vendor/urllib3-1.26.4.dist-info/METADATA": f"Name: urllib3\nVersion: 1.26.4\n{metadata}" + } + document: Final[dict[str, Any]] = json.loads(build_sbom("pip/example.py", vendored)) + assert [child["licenses"] for child in document["components"][0]["components"] if child["type"] == "library"] == [ + expected + ] + + +@pytest.mark.parametrize( + "requirement", + [ + pytest.param("urllib3", id="missing"), + pytest.param("urllib3>=1", id="range"), + pytest.param("urllib3==1.*", id="wildcard"), + pytest.param("urllib3 @ https://example.com/package.whl", id="url"), + ], +) +def test_sbom_rejects_unpinned_vendor(build_sbom: Callable[[str, dict[str, str]], str], requirement: str) -> None: + with pytest.raises(ValueError, match="Unpinned vendored dependency"): + build_sbom("pip/example.py", {"pip/_vendor/vendor.txt": requirement}) + + +@pytest.mark.parametrize( + ("epoch", "expected"), + [ + pytest.param(None, "2020-02-02T00:00:00+00:00", id="hatchling-default"), + pytest.param("1700000000", "2023-11-14T22:13:20+00:00", id="source-date-epoch"), + ], +) +def test_sbom_timestamp( + build_sbom: Callable[[str, dict[str, str]], str], monkeypatch: pytest.MonkeyPatch, epoch: str | None, expected: str +) -> None: + monkeypatch.delenv("SOURCE_DATE_EPOCH", raising=False) + if epoch is not None: + monkeypatch.setenv("SOURCE_DATE_EPOCH", epoch) + assert json.loads(build_sbom("pip/example.py", {}))["metadata"]["timestamp"] == expected + + +def test_sbom_release_notes_from_project_url(build_sbom: Callable[[str, dict[str, str]], str]) -> None: + root: Final[dict[str, Any]] = json.loads(build_sbom("pip/example.py", {}))["metadata"]["component"] + assert [reference for reference in root["externalReferences"] if reference["type"] == "release-notes"] == [ + {"type": "release-notes", "url": "https://example.com/changelog", "comment": "Project-URL: Changelog"}, + ] + + +def test_sbom_build_host_independent( + build_sbom: Callable[[str, dict[str, str]], str], + install_tool: Callable[[Path, str, bool, dict[str, bytes]], None], + mocker: MockerFixture, + tmp_path: Path, +) -> None: + sboms: Final[list[str]] = [] + for host, extension, installer, sys_version in ( + ( + { + "system": "Linux", + "release": "6.17.0-1022-azure", + "version": "#22~24.04.1-Ubuntu SMP", + "machine": "x86_64", + "platform": "Linux-6.17.0-1022-azure-x86_64-with-glibc2.39", + "python_compiler": "GCC 13.3.0", + "python_build": ("main", "Aug 5 2026 10:00:00"), + "freedesktop_os_release": {"ID": "ubuntu", "VERSION_ID": "24.04"}, + }, + "cpython-314-x86_64-linux-gnu.so", + b"uv\n", + "3.14.7 (main, Aug 5 2026, 10:00:00) [GCC 13.3.0]", + ), + ( + { + "system": "Linux", + "release": "7.0.12-linuxkit", + "version": "#1 SMP PREEMPT", + "machine": "aarch64", + "platform": "Linux-7.0.12-linuxkit-aarch64-with-glibc2.36", + "python_compiler": "Clang 20.1.4", + "python_build": ("main", "Sep 1 2026 08:00:00"), + "freedesktop_os_release": {"ID": "debian", "VERSION_ID": "12"}, + }, + "cpython-314-aarch64-linux-gnu.so", + b"pip\n", + "3.14.7 (main, Sep 1 2026, 08:00:00) [Clang 20.1.4]", + ), + ): + for name, value in host.items(): + mocker.patch(f"platform.{name}", autospec=True, return_value=value) + mocker.patch.object(sys, "version", sys_version) + install_tool(site := tmp_path / extension, "native", False, {f"native/_speedups.{extension}": b"\0"}) + install_tool(site, "pure", True, {"pure-1.0.dist-info/INSTALLER": installer}) + mocker.patch.object(sys, "path", [str(site), *sys.path]) + sboms.append(build_sbom("pip/example.py", {})) + mocker.stopall() + assert sboms[0] == sboms[1] + + +def test_sbom_ci_rerun(build_sbom: Callable[[str, dict[str, str]], str], monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("SOURCE_DATE_EPOCH", "1700000000") + monkeypatch.setenv("GITHUB_REPOSITORY", "downstream/project") + monkeypatch.setenv("GITHUB_SHA", "a" * 40) + monkeypatch.setenv("GITHUB_RUN_ID", "100") + monkeypatch.setenv("GITHUB_RUN_ATTEMPT", "1") + first: Final[str] = build_sbom("pip/example.py", {}) + monkeypatch.setenv("GITHUB_SHA", "b" * 40) + monkeypatch.setenv("GITHUB_RUN_ID", "200") + monkeypatch.setenv("GITHUB_RUN_ATTEMPT", "2") + assert build_sbom("pip/example.py", {}) == first + + +def test_sbom_spdx_bundled_wheel( + build_sbom: Callable[[str, dict[str, str]], str], render_spdx: Callable[[str], str] +) -> None: + cyclonedx: Final[str] = build_sbom("pip/example.py", {}) + assert json.loads(render_spdx(cyclonedx))["packages"][1] == { + "SPDXID": "SPDXRef-pkg-pypi-pip-1.0", + "checksums": [ + {"algorithm": "SHA256", "checksumValue": json.loads(cyclonedx)["components"][0]["hashes"][0]["content"]} + ], + "comment": "virtualenv:bundled-wheel: src/virtualenv/seed/wheels/embed/pip-1.0-py3-none-any.whl\n" + "virtualenv:seeded-for-python: 3.14", + "downloadLocation": "NOASSERTION", + "externalRefs": [ + {"referenceCategory": "PACKAGE-MANAGER", "referenceLocator": "pkg:pypi/pip@1.0", "referenceType": "purl"} + ], + "filesAnalyzed": False, + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "NOASSERTION", + "name": "pip", + "primaryPackagePurpose": "LIBRARY", + "versionInfo": "1.0", + } + + +def test_sbom_spdx_relationships( + build_sbom: Callable[[str, dict[str, str]], str], render_spdx: Callable[[str], str] +) -> None: + rendered: Final[str] = render_spdx(build_sbom("pip/example.py", {"pip/_vendor/vendor.txt": "urllib3==1.26.4\n"})) + assert [ + relationship + for relationship in json.loads(rendered)["relationships"] + if not relationship["spdxElementId"].startswith("SPDXRef-tool-") + ] == [ + { + "spdxElementId": "SPDXRef-DOCUMENT", + "relationshipType": "DESCRIBES", + "relatedSpdxElement": "SPDXRef-pkg-pypi-virtualenv-1.0", + }, + { + "spdxElementId": "SPDXRef-pkg-pypi-pip-1.0", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-pkg-pypi-pip-1.0-vendored-pkg-pypi-urllib3-1.26.4", + }, + { + "spdxElementId": "SPDXRef-pkg-pypi-virtualenv-1.0", + "relationshipType": "DEPENDS_ON", + "relatedSpdxElement": "SPDXRef-pkg-pypi-pip-1.0", + }, + { + "spdxElementId": "SPDXRef-pkg-pypi-virtualenv-1.0", + "relationshipType": "DEPENDS_ON", + "relatedSpdxElement": "SPDXRef-requires-dist-python-discovery-1.6", + }, + { + "spdxElementId": "SPDXRef-pkg-pypi-pip-1.0", + "relationshipType": "DEPENDS_ON", + "relatedSpdxElement": "SPDXRef-pkg-pypi-pip-1.0-vendored-pkg-pypi-urllib3-1.26.4", + }, + ] + + +def test_sbom_spdx_build_tools( + build_sbom: Callable[[str, dict[str, str]], str], render_spdx: Callable[[str], str] +) -> None: + assert { + relationship["relatedSpdxElement"] + for relationship in json.loads(render_spdx(build_sbom("pip/example.py", {})))["relationships"] + if relationship["relationshipType"] == "BUILD_TOOL_OF" + } == {"SPDXRef-pkg-pypi-virtualenv-1.0"} + + +@pytest.mark.parametrize( + ("metadata", "expected"), + [ + pytest.param("License-Expression: MIT\n", ("MIT", "NOASSERTION", None), id="expression"), + pytest.param( + "Classifier: License :: OSI Approved :: MIT License\nLicense: MIT\n", + ("NOASSERTION", "MIT", "Declared in package metadata as: MIT License; MIT"), + id="names", + ), + pytest.param( + "License: LGPLv3\n", + ("NOASSERTION", "NOASSERTION", "Declared in package metadata as: LGPLv3"), + id="unresolved-name", + ), + pytest.param("", ("NOASSERTION", "NOASSERTION", None), id="undeclared"), + ], +) +def test_sbom_spdx_declared_license( + build_sbom: Callable[[str, dict[str, str]], str], + render_spdx: Callable[[str], str], + metadata: str, + expected: tuple[str, str, str | None], +) -> None: + vendored: Final[dict[str, str]] = { + "pip/_vendor/urllib3-1.26.4.dist-info/METADATA": f"Name: urllib3\nVersion: 1.26.4\n{metadata}" + } + assert [ + (package["licenseDeclared"], package["licenseConcluded"], package.get("licenseComments")) + for package in json.loads(render_spdx(build_sbom("pip/example.py", vendored)))["packages"] + if (package["name"], package.get("versionInfo")) == ("urllib3", "1.26.4") + ] == [expected] + + +def test_sbom_spdx_creation_info( + build_sbom: Callable[[str, dict[str, str]], str], + render_spdx: Callable[[str], str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("SOURCE_DATE_EPOCH", "1700000000") + cyclonedx: Final[str] = build_sbom("pip/example.py", {}) + serial: Final[str] = json.loads(cyclonedx)["serialNumber"].removeprefix("urn:uuid:") + assert ( + (document := json.loads(render_spdx(cyclonedx)))["documentNamespace"], + document["creationInfo"]["created"], + ) == ( + f"https://github.com/pypa/virtualenv/sboms/virtualenv-1.0-{serial}", + "2023-11-14T22:13:20Z", + ) + + +def test_sbom_zipapp_components(zipapp_sbom: str) -> None: + assert [ + ( + component["purl"], + [prop["value"] for prop in component["properties"] if prop["name"].startswith("virtualenv:")], + [child["name"] for child in component["components"]], + ) + for component in json.loads(zipapp_sbom)["components"] + ] == [ + ( + "pkg:pypi/virtualenv@1.0", + [], + ["virtualenv-1.0.dist-info/METADATA", "virtualenv/__init__.py", "pip"], + ), + ( + "pkg:pypi/distlib@0.4", + ["__virtualenv__/distlib-0.4-py3-none-any", "3.14", "3.9"], + [ + "__virtualenv__/distlib-0.4-py3-none-any/distlib-0.4.dist-info/METADATA", + "__virtualenv__/distlib-0.4-py3-none-any/distlib/__init__.py", + ], + ), + ] + + +def test_sbom_zipapp_embedded_wheel(zipapp: Path, zipapp_sbom: str) -> None: + with zipfile.ZipFile(zipapp) as archive: + digest: Final[str] = hashlib.sha256( + archive.read("virtualenv/seed/wheels/embed/pip-1.0-py3-none-any.whl") + ).hexdigest() + assert json.loads(zipapp_sbom)["components"][0]["components"][2] == { + "type": "library", + "bom-ref": "pkg:pypi/pip@1.0", + "name": "pip", + "version": "1.0", + "purl": "pkg:pypi/pip@1.0", + "externalReferences": [], + "hashes": [{"alg": "SHA-256", "content": digest}], + "properties": [ + {"name": "virtualenv:bundled-wheel", "value": "virtualenv/seed/wheels/embed/pip-1.0-py3-none-any.whl"} + ], + } + + +def test_sbom_zipapp_loader_files(zipapp_sbom: str) -> None: + assert [ + (component["name"], component["hashes"][0]["content"]) + for component in json.loads(zipapp_sbom)["metadata"]["component"]["components"] + ] == [ + ("__main__.py", hashlib.sha256(b"# loader").hexdigest()), + ("distributions.json", hashlib.sha256(_DISTRIBUTIONS.encode()).hexdigest()), + ("modules.json", hashlib.sha256(b"{}").hexdigest()), + ] + + +def test_sbom_zipapp_embeds_document(zipapp: Path, zipapp_sbom: str) -> None: + with zipfile.ZipFile(zipapp) as archive: + assert archive.read("virtualenv.pyz.cdx.json").decode("utf-8") == zipapp_sbom + + +def test_sbom_zipapp_license_problems( + zipapp_sbom: str, license_problems: Callable[[list[dict[str, Any]]], list[str]] +) -> None: + assert license_problems(json.loads(zipapp_sbom)["components"]) == [ + "pkg:pypi/virtualenv@1.0: declares no license", + "pkg:pypi/pip@1.0: declares no license", + "pkg:pypi/distlib@0.4: declares no license", + ] + + +@pytest.mark.parametrize( + ("licenses", "expected"), + [ + pytest.param([_expression("MIT")], [], id="allowed"), + pytest.param([_expression("mit")], [], id="allowed-any-case"), + pytest.param([_expression("GPL-3.0-only")], [_denied("GPL-3.0-only")], id="denied"), + pytest.param([_expression("Apache-2.0 OR GPL-3.0-only")], [], id="or-one-allowed"), + pytest.param( + [_expression("GPL-3.0-only OR MPL-2.0")], [_denied("GPL-3.0-only OR MPL-2.0")], id="or-none-allowed" + ), + pytest.param([_expression("MIT AND BSD-3-Clause")], [], id="and-all-allowed"), + pytest.param([_expression("MIT AND GPL-3.0-only")], [_denied("MIT AND GPL-3.0-only")], id="and-one-denied"), + pytest.param([_expression("MIT OR GPL-3.0-only AND LGPL-3.0-only")], [], id="and-binds-tighter"), + pytest.param( + [_expression("(MIT OR GPL-3.0-only) AND LGPL-3.0-only")], + [_denied("(MIT OR GPL-3.0-only) AND LGPL-3.0-only")], + id="parentheses", + ), + pytest.param([_expression("Apache-2.0 WITH LLVM-exception")], [], id="with-allowed"), + pytest.param( + [_expression("GPL-2.0-only WITH Classpath-exception-2.0")], + [_denied("GPL-2.0-only WITH Classpath-exception-2.0")], + id="with-denied", + ), + pytest.param([_named("MIT License"), _concluded("MIT")], [], id="concluded-allowed"), + pytest.param([_named("MPL 2.0"), _concluded("MPL-2.0")], [_denied("MPL-2.0")], id="concluded-denied"), + pytest.param( + [_concluded("MIT"), _concluded("LGPL-3.0-only")], [_denied("LGPL-3.0-only")], id="concluded-all-apply" + ), + pytest.param( + [_named("MIT License")], + [ + ( + "pkg:pypi/example@1.0: declares license names ['MIT License'] with no SPDX id; map them in" + " hatch_build.py _CLASSIFIER_LICENSES" + ) + ], + id="name-only", + ), + pytest.param([], ["pkg:pypi/example@1.0: declares no license"], id="undeclared"), + ], +) +def test_sbom_license_policy( + license_problems: Callable[[list[dict[str, Any]]], list[str]], + licenses: list[dict[str, Any]], + expected: list[str], +) -> None: + component: Final[dict[str, Any]] = { + "type": "library", + "bom-ref": "pkg:pypi/example@1.0", + "hashes": [{"alg": "SHA-256", "content": "0" * 64}], + "licenses": licenses, + } + assert license_problems([component]) == expected + + +@pytest.mark.parametrize( + ("expression", "reason"), + [ + pytest.param("MIT AND", "it ends early", id="trailing-operator"), + pytest.param("OR MIT", "unexpected 'OR'", id="leading-operator"), + pytest.param("(MIT", "it ends early", id="unclosed"), + pytest.param("MIT)", "unexpected ')'", id="unopened"), + pytest.param("(MIT Apache-2.0)", "expected ')', got 'Apache-2.0'", id="missing-operator"), + pytest.param("MIT and Apache-2.0", "unexpected 'and'", id="lower-case-operator"), + pytest.param("Apache-2.0 WITH OR", "unexpected 'OR'", id="with-operator"), + ], +) +def test_sbom_license_policy_malformed( + license_problems: Callable[[list[dict[str, Any]]], list[str]], expression: str, reason: str +) -> None: + component: Final[dict[str, Any]] = { + "type": "library", + "bom-ref": "pkg:pypi/example@1.0", + "hashes": [{"alg": "SHA-256", "content": "0" * 64}], + "licenses": [_expression(expression)], + } + assert license_problems([component]) == [ + f"pkg:pypi/example@1.0: license expression {expression!r} is malformed: {reason}" + ] + + +@pytest.mark.parametrize( + "component", + [ + pytest.param( + {"type": "library", "bom-ref": "requires-dist:distlib", "name": "distlib"}, id="declared-requirement" + ), + pytest.param( + { + "type": "library", + "bom-ref": "pkg:pypi/pip@1.0", + "hashes": [{"alg": "SHA-256", "content": "0" * 64}], + "licenses": [_expression("MIT")], + "components": [ + {"type": "library", "bom-ref": "pkg:pypi/pip@1.0#vendored/pkg:pypi/autocommand@2.2.2"}, + ], + }, + id="vendored-by-seed-wheel", + ), + pytest.param( + {"type": "file", "bom-ref": "loader", "hashes": [{"alg": "SHA-256", "content": "0" * 64}]}, id="file" + ), + ], +) +def test_sbom_license_policy_skips_unshipped( + license_problems: Callable[[list[dict[str, Any]]], list[str]], component: dict[str, Any] +) -> None: + assert license_problems([component]) == [] + + +def test_sbom_license_policy_judges_files_owner( + license_problems: Callable[[list[dict[str, Any]]], list[str]], +) -> None: + component: Final[dict[str, Any]] = { + "type": "library", + "bom-ref": "pkg:pypi/distlib@0.4", + "licenses": [_concluded("LGPL-3.0-only")], + "components": [{"type": "file", "bom-ref": "pkg:pypi/distlib@0.4#distlib/__init__.py"}], + } + assert license_problems([component]) == [_denied("LGPL-3.0-only", "pkg:pypi/distlib@0.4")] + + +@pytest.fixture +def license_problems() -> Callable[[list[dict[str, Any]]], list[str]]: + return runpy.run_path(str(Path(__file__).parents[2] / "tasks" / "license_policy.py"))["license_problems"] + + +@pytest.fixture +def zipapp(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + wheel: Final[BytesIO] = BytesIO() + with zipfile.ZipFile(wheel, "w") as archive: + archive.writestr("pip-1.0.dist-info/METADATA", "Metadata-Version: 2.4\nName: pip\nVersion: 1.0\n") + bundled: Final[str] = "__virtualenv__/distlib-0.4-py3-none-any" + with zipfile.ZipFile(pyz := tmp_path / "virtualenv.pyz", "w") as archive: + archive.writestr("__main__.py", "# loader") + archive.writestr("modules.json", "{}") + archive.writestr("distributions.json", _DISTRIBUTIONS) + archive.writestr("virtualenv-1.0.dist-info/METADATA", "Metadata-Version: 2.4\nName: virtualenv\nVersion: 1.0\n") + archive.writestr("virtualenv/__init__.py", "") + archive.writestr("virtualenv/seed/wheels/embed/pip-1.0-py3-none-any.whl", wheel.getvalue()) + archive.writestr( + f"{bundled}/distlib-0.4.dist-info/METADATA", "Metadata-Version: 2.4\nName: distlib\nVersion: 0.4\n" + ) + archive.writestr(f"{bundled}/distlib/__init__.py", "") + monkeypatch.syspath_prepend(str(Path(__file__).parents[2])) + monkeypatch.setattr(sys, "argv", ["zipapp_sbom.py", str(pyz), str(tmp_path / "virtualenv.pyz.cdx.json")]) + runpy.run_path(str(Path(__file__).parents[2] / "tasks" / "zipapp_sbom.py"), run_name="__main__") + return pyz + + +@pytest.fixture +def zipapp_sbom(zipapp: Path) -> str: + return (zipapp.parent / "virtualenv.pyz.cdx.json").read_text(encoding="utf-8") + + +@pytest.fixture +def render_spdx(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Callable[[str], str]: + def render(cyclonedx: str) -> str: + (source := tmp_path / "virtualenv.cdx.json").write_text(cyclonedx, encoding="utf-8") + monkeypatch.setattr( + sys, "argv", ["cyclonedx_to_spdx.py", str(source), str(target := tmp_path / "out.spdx.json")] + ) + runpy.run_path(str(Path(__file__).parents[2] / "tasks" / "cyclonedx_to_spdx.py"), run_name="__main__") + return target.read_text(encoding="utf-8") + + return render + + +@pytest.fixture +def install_tool() -> Callable[[Path, str, bool, dict[str, bytes]], None]: + def install(site: Path, name: str, purelib: bool, files: dict[str, bytes]) -> None: + record: Final[StringIO] = StringIO(newline="") + for path, content in files.items(): + (target := site / path).parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(content) + digest = base64.urlsafe_b64encode(hashlib.sha256(content).digest()).rstrip(b"=").decode() + csv.writer(record).writerow((path, f"sha256={digest}", len(content))) + (dist_info := site / f"{name}-1.0.dist-info").mkdir(exist_ok=True) + (dist_info / "METADATA").write_text(f"Metadata-Version: 2.4\nName: {name}\nVersion: 1.0\n", encoding="utf-8") + (dist_info / "WHEEL").write_text( + f"Wheel-Version: 1.0\nRoot-Is-Purelib: {str(purelib).lower()}\n", encoding="utf-8" + ) + (dist_info / "RECORD").write_text(record.getvalue(), encoding="utf-8", newline="") + + return install + + +@pytest.fixture +def build_sbom(tmp_path: Path) -> Callable[[str, dict[str, str]], str]: + shutil.copyfile(Path(__file__).parents[2] / "hatch_build.py", tmp_path / "hatch_build.py") + (tmp_path / "LICENSE").write_text("Copyright (c) example\n", encoding="utf-8") + embed: Final[Path] = tmp_path / "src" / "virtualenv" / "seed" / "wheels" / "embed" + embed.mkdir(parents=True) + (embed / "__init__.py").write_text( + 'BUNDLE_SUPPORT = {"3.14": {"pip": "pip-1.0-py3-none-any.whl"}}\n', encoding="utf-8" + ) + + def build(filename: str, vendored_files: dict[str, str]) -> str: + record: Final[StringIO] = StringIO(newline="") + csv.writer(record).writerows([ + (filename, "sha256=ungWv48Bz-pBQUDeXa4iI7ADYaOWF3qctBD_YfIAFa0", "3"), + ("pip-1.0.dist-info/RECORD", "", ""), + ]) + with zipfile.ZipFile(embed / "pip-1.0-py3-none-any.whl", "w") as archive: + for name, content in vendored_files.items(): + archive.writestr(zipfile.ZipInfo(name), content) + archive.writestr( + zipfile.ZipInfo("pip-1.0.dist-info/METADATA"), "Metadata-Version: 2.4\nName: pip\nVersion: 1.0\n" + ) + archive.writestr(zipfile.ZipInfo("pip-1.0.dist-info/RECORD"), record.getvalue()) + archive.writestr(zipfile.ZipInfo(filename), b"abc") + builder: Final[WheelBuilder] = WheelBuilder( + str(tmp_path), + config={ + "project": { + "name": "virtualenv", + "version": "1.0", + "description": "example", + "requires-python": ">=3.9", + "license": "MIT", + "maintainers": [{"name": "Example"}], + "dependencies": ["python-discovery>=1.6"], + "urls": {"Changelog": "https://example.com/changelog"}, + }, + "tool": {"hatch": {"build": {"hooks": {"custom": {"path": "hatch_build.py"}}}}}, + }, + ) + build_data: Final[dict[str, Any]] = builder.get_default_build_data() + builder.get_build_hooks(str(tmp_path))["custom"].initialize("standard", build_data) + return Path(build_data["sbom_files"][0]).read_text(encoding="utf-8") + + return build diff --git a/tests/tasks/test_wheel_age.py b/tests/tasks/test_wheel_age.py new file mode 100644 index 000000000..2f6d96767 --- /dev/null +++ b/tests/tasks/test_wheel_age.py @@ -0,0 +1,462 @@ +from __future__ import annotations + +import hashlib +import json +import runpy +import shutil +import subprocess +import sys +import zipfile +from io import BytesIO +from pathlib import Path +from typing import TYPE_CHECKING, Final +from urllib.error import URLError + +import pytest + +if TYPE_CHECKING: + from collections.abc import Callable + from urllib.request import Request + + from pytest_mock import MockerFixture + + +@pytest.fixture +def wheel_repo(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, time_freeze: Callable[[str], None]) -> Path: + monkeypatch.chdir(tmp_path) + time_freeze("2026-09-21T00:00:00+00:00") + embed: Final[Path] = tmp_path / "src" / "virtualenv" / "seed" / "wheels" / "embed" + embed.mkdir(parents=True) + (embed / "pip-1-py3-none-any.whl").write_bytes(b"old") + subprocess.run(["git", "init", "--quiet"], check=True) + subprocess.run(["git", "add", "."], check=True) + subprocess.run( + ["git", "-c", "user.name=Test", "-c", "user.email=test@example.com", "commit", "-qm", "initial"], check=True + ) + return embed + + +@pytest.fixture +def check_age(capsys: pytest.CaptureFixture[str]) -> Callable[[], str]: + def invoke() -> str: + runpy.run_path(str(Path(__file__).parents[2] / "tasks" / "check_wheel_age.py"), run_name="__main__") + return capsys.readouterr().out + + return invoke + + +@pytest.mark.parametrize( + "wheel", + [ + pytest.param(("pip-2-py3-none-any.whl", "pip/2"), id="simple-name"), + pytest.param(("importlib_metadata-8.7.0-py3-none-any.whl", "importlib_metadata/8.7.0"), id="normalized-name"), + pytest.param(("importlib_metadata-8.7.0-1-py3-none-any.whl", "importlib_metadata/8.7.0"), id="build-tag"), + ], +) +@pytest.mark.parametrize( + "age", + [ + pytest.param(("2026-09-13T00:00:00Z", "skip=false\n"), id="old"), + pytest.param(("2026-09-14T00:00:00+00:00", "skip=false\n"), id="seven-days"), + pytest.param(("2026-09-15T00:00:00Z", "skip=true\n"), id="new"), + ], +) +def test_new_wheel_age( + wheel_repo: Path, + mocker: MockerFixture, + check_age: Callable[[], str], + age: tuple[str, str], + wheel: tuple[str, str], +) -> None: + (wheel_repo / "pip-1-py3-none-any.whl").unlink() + (wheel_repo / wheel[0]).write_bytes(b"new") + request = mocker.patch( + "urllib.request.urlopen", + autospec=True, + return_value=BytesIO( + json.dumps({ + "urls": [ + {"filename": "pip-2.tar.gz", "upload_time_iso_8601": "2020-01-01T00:00:00Z"}, + {"filename": wheel[0], "upload_time_iso_8601": age[0]}, + ] + }).encode() + ), + ) + assert check_age() == age[1] + request.assert_called_once_with(f"https://pypi.org/pypi/{wheel[1]}/json", timeout=30) + + +@pytest.mark.parametrize("staged", [pytest.param(False, id="untracked"), pytest.param(True, id="staged")]) +def test_missing_wheel_metadata( + wheel_repo: Path, mocker: MockerFixture, check_age: Callable[[], str], staged: bool +) -> None: + (wheel_repo / "pip-2-py3-none-any.whl").write_bytes(b"new") + if staged: + subprocess.run(["git", "add", "."], check=True) + mocker.patch("urllib.request.urlopen", autospec=True, return_value=BytesIO(b'{"urls": []}')) + with pytest.raises(SystemExit, match=r"PyPI has no upload information for pip-2-py3-none-any\.whl"): + check_age() + + +def test_metadata_network_failure(wheel_repo: Path, mocker: MockerFixture, check_age: Callable[[], str]) -> None: + (wheel_repo / "pip-2-py3-none-any.whl").write_bytes(b"new") + mocker.patch("urllib.request.urlopen", autospec=True, side_effect=URLError("unavailable")) + with pytest.raises(URLError, match="unavailable"): + check_age() + + +def test_removed_wheels_need_no_age_check(wheel_repo: Path, check_age: Callable[[], str]) -> None: + (wheel_repo / "pip-1-py3-none-any.whl").unlink() + assert check_age() == "skip=false\n" + + +def test_missing_git(monkeypatch: pytest.MonkeyPatch, check_age: Callable[[], str]) -> None: + monkeypatch.setenv("PATH", "") + with pytest.raises(SystemExit, match="git is required"): + check_age() + + +@pytest.fixture +def generator_repo(tmp_path: Path) -> Path: + root: Final[Path] = Path(__file__).parents[2] + (tmp_path / "tasks").mkdir() + shutil.copyfile(root / "tasks" / "upgrade_wheels.py", tmp_path / "tasks" / "upgrade_wheels.py") + shutil.copyfile(root / ".pre-commit-config.yaml", tmp_path / ".pre-commit-config.yaml") + shutil.copyfile(root / "pyproject.toml", tmp_path / "pyproject.toml") + embed: Final[Path] = tmp_path / "src" / "virtualenv" / "seed" / "wheels" / "embed" + embed.mkdir(parents=True) + (embed / "__init__.py").write_text( + 'BUNDLE_SUPPORT = {"3.14": {"pip": "pip-1-py3-none-any.whl"}}\n', encoding="utf-8" + ) + with zipfile.ZipFile(embed / "pip-1-py3-none-any.whl", "w") as archive: + archive.writestr("pip-1.dist-info/METADATA", "Name: pip\nVersion: 1\n") + archive.writestr("pip-1.dist-info/licenses/LICENSE.txt", "Copyright Example\nPermission to redistribute.\n") + subprocess.run(["git", "init", "--quiet"], cwd=tmp_path, check=True) + return tmp_path + + +@pytest.fixture +def regenerate(generator_repo: Path) -> Callable[[], subprocess.CompletedProcess[str]]: + def invoke() -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, "tasks/upgrade_wheels.py", "--regen"], + cwd=generator_repo, + capture_output=True, + text=True, + encoding="utf-8", + check=True, + ) + + return invoke + + +def test_generator_converges(generator_repo: Path, regenerate: Callable[[], subprocess.CompletedProcess[str]]) -> None: + regenerate() + assert "Copyright Example" in (generator_repo / "THIRD-PARTY-NOTICES.md").read_text(encoding="utf-8") + subprocess.run(["git", "add", "."], cwd=generator_repo, check=True) + regenerate() + subprocess.run( + ["pre-commit", "run", "mdformat", "--files", "THIRD-PARTY-NOTICES.md"], + cwd=generator_repo, + check=True, + ) + assert ( + subprocess.run(["git", "diff", "--exit-code"], cwd=generator_repo, capture_output=True, check=False).returncode + == 0 + ) + + +def test_generator_updates_license( + generator_repo: Path, regenerate: Callable[[], subprocess.CompletedProcess[str]] +) -> None: + regenerate() + subprocess.run(["git", "add", "."], cwd=generator_repo, check=True) + wheel: Final[Path] = generator_repo / "src/virtualenv/seed/wheels/embed/pip-1-py3-none-any.whl" + with zipfile.ZipFile(wheel, "w") as archive: + archive.writestr("pip-1.dist-info/METADATA", "Name: pip\nVersion: 1\n") + archive.writestr("pip-1.dist-info/licenses/LICENSE.txt", "Copyright Changed\nPermission to redistribute.\n") + regenerate() + assert "Copyright Changed" in (generator_repo / "THIRD-PARTY-NOTICES.md").read_text(encoding="utf-8") + + +def test_generator_formatter_failure( + generator_repo: Path, regenerate: Callable[[], subprocess.CompletedProcess[str]] +) -> None: + (generator_repo / ".pre-commit-config.yaml").write_text("repos: invalid\n", encoding="utf-8") + with pytest.raises(subprocess.CalledProcessError): + regenerate() + + +@pytest.fixture +def apply_patch(wheel_repo: Path) -> Callable[[str, str], subprocess.CompletedProcess[str]]: + def invoke(filename: str, mode: str = "100644") -> subprocess.CompletedProcess[str]: + blob: Final[str] = subprocess.run( + ["git", "hash-object", "-w", "--stdin"], + input="new\0content", + capture_output=True, + text=True, + encoding="utf-8", + check=True, + ).stdout.strip() + if mode == "000000": + subprocess.run(["git", "update-index", "--force-remove", filename], check=True) + else: + subprocess.run(["git", "update-index", "--add", "--cacheinfo", mode, blob, filename], check=True) + patch: Final[bytes] = subprocess.run( + ["git", "diff", "--cached", "--binary"], capture_output=True, check=True + ).stdout + subprocess.run(["git", "reset", "-q", "HEAD", "--", filename], check=True) + patch_file: Final[Path] = wheel_repo.parents[4] / "upgrade.patch" + patch_file.write_bytes(patch) + return subprocess.run( + [sys.executable, str(Path(__file__).parents[2] / "tasks" / "apply_upgrade_patch.py"), str(patch_file)], + capture_output=True, + text=True, + encoding="utf-8", + check=False, + ) + + return invoke + + +@pytest.mark.parametrize( + "filename", + [ + pytest.param("THIRD-PARTY-NOTICES.md", id="notices"), + pytest.param("docs/changelog/u.bugfix.rst", id="changelog"), + pytest.param("pylock.zipapp.toml", id="zipapp-lock"), + pytest.param("src/virtualenv/seed/wheels/embed/__init__.py", id="bundle-index"), + pytest.param("src/virtualenv/seed/wheels/embed/pip-2-py3-none-any.whl", id="wheel"), + pytest.param("tasks/ci-tools.json", id="ci-tools"), + pytest.param("tasks/release-requirements.txt", id="release-lock"), + ], +) +def test_upgrade_patch_applies( + apply_patch: Callable[[str, str], subprocess.CompletedProcess[str]], filename: str +) -> None: + result: Final[subprocess.CompletedProcess[str]] = apply_patch(filename, "100644") + assert (result.returncode, Path(filename).read_bytes()) == (0, b"new\0content") + + +@pytest.mark.parametrize( + ("filename", "mode"), + [ + pytest.param(".github/workflows/release.yaml", "100644", id="workflow"), + pytest.param("tasks/apply_upgrade_patch.py", "100644", id="validator"), + pytest.param("src/virtualenv/seed/wheels/embed/subdir/pip.whl", "100644", id="nested-path"), + pytest.param("THIRD-PARTY-NOTICES.md", "120000", id="symlink"), + pytest.param("THIRD-PARTY-NOTICES.md", "100755", id="executable"), + ], +) +def test_upgrade_patch_rejects( + apply_patch: Callable[[str, str], subprocess.CompletedProcess[str]], filename: str, mode: str +) -> None: + result: Final[subprocess.CompletedProcess[str]] = apply_patch(filename, mode) + assert (result.returncode, Path(filename).exists()) == (1, False) + assert "unsupported path or file mode" in result.stderr + + +def test_upgrade_patch_removes_old_wheel( + wheel_repo: Path, apply_patch: Callable[[str, str], subprocess.CompletedProcess[str]] +) -> None: + result: Final[subprocess.CompletedProcess[str]] = apply_patch( + "src/virtualenv/seed/wheels/embed/pip-1-py3-none-any.whl", "000000" + ) + assert (result.returncode, (wheel_repo / "pip-1-py3-none-any.whl").exists()) == (0, False) + + +def test_upgrade_patch_missing_git(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("PATH", "") + with pytest.raises(SystemExit, match="git is required"): + runpy.run_path(str(Path(__file__).parents[2] / "tasks" / "apply_upgrade_patch.py"), run_name="__main__") + + +_NU_ASSET: Final[str] = "nu-0.116.0-x86_64-pc-windows-msvc.zip" +_RUSTPYTHON_ASSETS: Final[dict[str, str]] = { + "Linux": "rustpython-release-Linux-x86_64-unknown-linux-gnu", + "macOS": "rustpython-release-macOS-aarch64-apple-darwin", + "Windows": "rustpython-release-Windows-x86_64-pc-windows-msvc.exe", +} +_OLD_PINS: Final[dict[str, dict[str, str | dict[str, dict[str, str]]]]] = { + "graalpy": {"tag": "graal-24.1.2", "assets": {}}, + "mermaid": {"version": "11.12.1"}, + "nushell": {"tag": "0.115.1", "assets": {"Windows": {"name": "nu-old.zip", "sha256": "old"}}}, + "rustpython": {"tag": "old", "assets": {"Linux": {"name": "rp-old", "sha256": "old"}}}, +} + + +@pytest.fixture +def ci_tools(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("GH_TOKEN", "token") + (tmp_path / "tasks").mkdir() + pins: Final[Path] = tmp_path / "tasks" / "ci-tools.json" + pins.write_text(json.dumps(_OLD_PINS), encoding="utf-8") + return pins + + +@pytest.fixture +def npm_registry(time_freeze: Callable[[str], None]) -> dict[str, dict[str, object]]: + time_freeze("2026-09-21T00:00:00+00:00") + return {"time": {"created": "2014-01-01T00:00:00.000Z"}, "versions": {}} + + +@pytest.fixture +def github( + mocker: MockerFixture, npm_registry: dict[str, dict[str, object]] +) -> Callable[[str, dict[str, bytes], dict[str, str]], None]: + def serve(nushell_tag: str, contents: dict[str, bytes], digests: dict[str, str]) -> None: + names: Final[list[str]] = sorted(contents) + + def respond(request: Request, **_kwargs: int) -> BytesIO: + url: Final[str] = request.full_url + if url == "https://registry.npmjs.org/mermaid": + return BytesIO(json.dumps(npm_registry).encode()) + if url == "https://api.github.com/graphql": + repository: Final[str] = json.loads(request.data)["variables"]["name"] + return BytesIO( + json.dumps({ + "data": { + "repository": { + "latestRelease": { + "tagName": "graal-25.4.4" if repository == "graalpython" else nushell_tag + }, + "releases": {"nodes": [{"tagName": "2026-09-20-main-1"}]}, + } + } + }).encode() + ) + if "/releases/tags/" in url: + return BytesIO( + json.dumps({ + "assets": [ + {"id": index, "name": name, "digest": digests.get(name)} for index, name in enumerate(names) + ] + }).encode() + ) + return BytesIO(contents[names[int(url.rsplit("/", 1)[1])]]) + + mocker.patch("urllib.request.urlopen", autospec=True, side_effect=respond) + + return serve + + +@pytest.fixture +def upgrade_ci_tools() -> Callable[[], None]: + def invoke() -> None: + runpy.run_path(str(Path(__file__).parents[2] / "tasks" / "upgrade_ci_tools.py"), run_name="__main__") + + return invoke + + +def test_ci_tools_pin_newest_releases( + ci_tools: Path, + github: Callable[[str, dict[str, bytes], dict[str, str]], None], + upgrade_ci_tools: Callable[[], None], +) -> None: + github( + "0.116.0", + {_NU_ASSET: b"nu", **{name: name.encode() for name in _RUSTPYTHON_ASSETS.values()}}, + {_NU_ASSET: f"sha256:{hashlib.sha256(b'nu').hexdigest()}"}, + ) + upgrade_ci_tools() + assert json.loads(ci_tools.read_text(encoding="utf-8")) == { + "graalpy": {"tag": "graal-25.4.4", "assets": {}}, + "mermaid": {"version": "11.12.1"}, + "nushell": { + "tag": "0.116.0", + "assets": {"Windows": {"name": _NU_ASSET, "sha256": hashlib.sha256(b"nu").hexdigest()}}, + }, + "rustpython": { + "tag": "2026-09-20-main-1", + "assets": { + runner_os: {"name": name, "sha256": hashlib.sha256(name.encode()).hexdigest()} + for runner_os, name in _RUSTPYTHON_ASSETS.items() + }, + }, + } + + +@pytest.mark.parametrize( + "release", + [ + pytest.param(("11.9.0", "2026-09-10T00:00:00.000Z", {}), id="older-line-published-later"), + pytest.param(("12.0.0-rc.1", "2026-09-01T00:00:00.000Z", {}), id="prerelease"), + pytest.param(("12.0.0", "2026-09-15T00:00:00.000Z", {}), id="inside-cooldown"), + pytest.param(("12.0.0", "2026-09-01T00:00:00.000Z", {"deprecated": "broken"}), id="deprecated"), + pytest.param(("12.0.0", "2026-09-01T00:00:00.000Z", None), id="unpublished"), + ], +) +def test_ci_tools_pin_newest_mermaid( + ci_tools: Path, + github: Callable[[str, dict[str, bytes], dict[str, str]], None], + upgrade_ci_tools: Callable[[], None], + npm_registry: dict[str, dict[str, object]], + release: tuple[str, str, dict[str, str] | None], +) -> None: + version, published, metadata = release + npm_registry["time"] |= {"11.13.0": "2026-08-01T00:00:00.000Z", version: published} + npm_registry["versions"] |= {"11.13.0": {}} | ({} if metadata is None else {version: metadata}) + github("0.115.1", dict.fromkeys(_RUSTPYTHON_ASSETS.values(), b"rp"), {}) + upgrade_ci_tools() + assert json.loads(ci_tools.read_text(encoding="utf-8"))["mermaid"] == {"version": "11.13.0"} + + +def test_ci_tools_keep_newer_mermaid_pin( + ci_tools: Path, + github: Callable[[str, dict[str, bytes], dict[str, str]], None], + upgrade_ci_tools: Callable[[], None], + npm_registry: dict[str, dict[str, object]], +) -> None: + npm_registry["time"] |= {"11.0.0": "2024-08-01T00:00:00.000Z"} + npm_registry["versions"] |= {"11.0.0": {}} + github("0.115.1", dict.fromkeys(_RUSTPYTHON_ASSETS.values(), b"rp"), {}) + upgrade_ci_tools() + assert json.loads(ci_tools.read_text(encoding="utf-8"))["mermaid"] == _OLD_PINS["mermaid"] + + +def test_ci_tools_keep_hashes_of_unchanged_tag( + ci_tools: Path, + github: Callable[[str, dict[str, bytes], dict[str, str]], None], + upgrade_ci_tools: Callable[[], None], +) -> None: + github("0.115.1", dict.fromkeys(_RUSTPYTHON_ASSETS.values(), b"replaced"), {}) + upgrade_ci_tools() + assert json.loads(ci_tools.read_text(encoding="utf-8"))["nushell"] == _OLD_PINS["nushell"] + + +@pytest.mark.parametrize( + ("release", "error"), + [ + pytest.param( + ({_NU_ASSET: b"nu"}, {_NU_ASSET: "sha256:0000"}), + rf"{_NU_ASSET} hashes to \w+ but GitHub reports sha256:0000", + id="digest-mismatch", + ), + pytest.param( + ({"nu-0.116.0-aarch64-pc-windows-msvc.zip": b"nu"}, {}), + rf"nushell/nushell 0\.116\.0 has no asset {_NU_ASSET}", + id="missing-asset", + ), + ], +) +def test_ci_tools_reject_release( + ci_tools: Path, + github: Callable[[str, dict[str, bytes], dict[str, str]], None], + upgrade_ci_tools: Callable[[], None], + release: tuple[dict[str, bytes], dict[str, str]], + error: str, +) -> None: + github("0.116.0", *release) + with pytest.raises(SystemExit, match=error): + upgrade_ci_tools() + assert json.loads(ci_tools.read_text(encoding="utf-8")) == _OLD_PINS + + +def test_ci_tools_require_token( + ci_tools: Path, monkeypatch: pytest.MonkeyPatch, upgrade_ci_tools: Callable[[], None] +) -> None: + monkeypatch.delenv("GH_TOKEN") + with pytest.raises(SystemExit, match="GH_TOKEN is required"): + upgrade_ci_tools() + assert json.loads(ci_tools.read_text(encoding="utf-8")) == _OLD_PINS diff --git a/tests/types.py b/tests/types.py new file mode 100644 index 000000000..0466460b2 --- /dev/null +++ b/tests/types.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from typing import Protocol + + +class _VersionInfo(Protocol): + major: int + minor: int + + +class Interpreter(Protocol): + prefix: str + system_prefix: str + system_executable: str + free_threaded: bool + version_info: _VersionInfo + sysconfig_vars: dict[str, object] + + +class MakeInterpreter(Protocol): + def __call__( + self, + sysconfig_vars: dict[str, object] | None = ..., + prefix: str = ..., + free_threaded: bool = ..., + version_info: tuple[int, ...] = ..., + ) -> Interpreter: ... diff --git a/tests/unit/activation/conftest.py b/tests/unit/activation/conftest.py index 53a819f96..5ab576a2f 100644 --- a/tests/unit/activation/conftest.py +++ b/tests/unit/activation/conftest.py @@ -7,11 +7,20 @@ from os.path import dirname, normcase from pathlib import Path from subprocess import Popen +from typing import Final, NamedTuple import pytest from virtualenv.run import cli_run +_STATE_VARS: Final[tuple[str, ...]] = ( + "VIRTUAL_ENV", + "VIRTUAL_ENV_PROMPT", + "TCL_LIBRARY", + "TK_LIBRARY", + "PKG_CONFIG_PATH", +) + class ActivationTester: def __init__(self, of_class, session, cmd, activate_script, extension) -> None: @@ -34,12 +43,13 @@ def get_version(self, raise_on_fail): try: process = Popen( self._version_cmd, + stdin=subprocess.DEVNULL, universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8", ) - out, err = process.communicate() + out, err = process.communicate(timeout=30) except Exception as exception: self._version = exception if raise_on_fail: @@ -75,14 +85,22 @@ def __call__(self, monkeypatch, tmp_path): invoke, env = [*self._invoke_script, str(test_script)], self.env(tmp_path) try: - process = Popen(invoke, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env) - raw_, _ = process.communicate() - raw = raw_.decode(errors="replace") - assert process.returncode == 0, raw + process = Popen(invoke, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env) + raw_, _ = process.communicate(timeout=60) + except subprocess.TimeoutExpired as exc: + process.kill() + remaining, _ = process.communicate(timeout=5) + partial = (exc.stdout or b"") + (remaining or b"") + pytest.fail( + f"Activation script timed out:\nPartial output:\n{partial.decode(errors='replace')}\nCommand: {invoke}" + ) except subprocess.CalledProcessError as exception: output = exception.output + exception.stderr - assert not exception.returncode, output # noqa: PT017 + assert not exception.returncode, output # ruff:ignore[pytest-assert-in-except] return None + else: + raw = raw_.decode(errors="replace") + assert process.returncode == 0, raw out = re.sub(r"pydev debugger: process \d+ is connecting\n\n", "", raw, flags=re.MULTILINE).strip().splitlines() self.assert_output(out, raw, tmp_path) @@ -91,14 +109,14 @@ def __call__(self, monkeypatch, tmp_path): def non_source_activate(self, activate_script): return [*self._invoke_script, str(activate_script)] - def env(self, tmp_path): # noqa: ARG002 + def env(self, tmp_path): # ruff:ignore[unused-method-argument] env = os.environ.copy() # add the current python executable folder to the path so we already have another python on the path # also keep the path so the shells (fish, bash, etc can be discovered) env["PYTHONIOENCODING"] = "utf-8" env["PATH"] = os.pathsep.join([dirname(sys.executable), *env.get("PATH", "").split(os.pathsep)]) # clear up some environment variables so they don't affect the tests - for key in [k for k in env if k.startswith(("_OLD", "VIRTUALENV_"))]: + for key in [k for k in env if k.startswith(("_OLD", "VIRTUALENV_", "COVERAGE_"))]: del env[key] return env @@ -111,59 +129,83 @@ def _generate_test_script(self, activate_script, tmp_path): return test_script def _get_test_lines(self, activate_script): - return [ - self.print_python_exe(), - self.print_os_env_var("VIRTUAL_ENV"), - self.print_os_env_var("VIRTUAL_ENV_PROMPT"), + steps = [ + self.print_state(_STATE_VARS), self.activate_call(activate_script), - self.print_python_exe(), - self.print_os_env_var("VIRTUAL_ENV"), - self.print_os_env_var("VIRTUAL_ENV_PROMPT"), + self.print_state(_STATE_VARS), self.print_prompt(), # \\ loads documentation from the virtualenv site packages self.pydoc_call, self.deactivate, - self.print_python_exe(), - self.print_os_env_var("VIRTUAL_ENV"), - self.print_os_env_var("VIRTUAL_ENV_PROMPT"), + self.print_state(_STATE_VARS), "", # just finish with an empty new line ] + result = [] + for index, step in enumerate(steps): + result.extend((self.print_marker(index), step)) + return result + + def print_marker(self, step) -> str: + return f'echo "__STEP_{step}__"' - def assert_output(self, out, raw, tmp_path): + def assert_output(self, out, raw, tmp_path) -> None: """Compare _get_test_lines() with the expected values.""" + out = [line for line in out if not line.strip('"').startswith("__STEP_")] assert out[0], raw assert out[1] == "None", raw assert out[2] == "None", raw + self.assert_tcl_tk_library(out[3:5], out[9:11], out[-3:-1], raw) + self.assert_pkg_config_path(out[5], out[11], out[-1], raw) # self.activate_call(activate_script) runs at this point python_exe = self._creator.exe.parent / os.path.basename(sys.executable) - assert self.norm_path(out[3]) == self.norm_path(python_exe), raw - assert self.norm_path(out[4]) == self.norm_path(self._creator.dest).replace("\\\\", "\\"), raw - assert out[5] == self._creator.env_name + assert self.norm_path(out[6]) == self.norm_path(python_exe), raw + assert self.norm_path(out[7]) == self.norm_path(self._creator.dest).replace("\\\\", "\\"), raw + assert out[8] == self._creator.env_name # Some attempts to test the prompt output print more than 1 line. # So we need to check if the prompt exists on any of them. prompt_text = f"({self._creator.env_name}) " - assert any(prompt_text in line for line in out[6:-4]), raw + assert any(prompt_text in line for line in out[12:-7]), raw - assert out[-4] == "wrote pydoc_test.html", raw + assert out[-7] == "wrote pydoc_test.html", raw content = tmp_path / "pydoc_test.html" assert content.exists(), raw # post deactivation, same as before - assert out[-3] == out[0], raw - assert out[-2] == "None", raw - assert out[-1] == "None", raw + assert out[-6] == out[0], raw + assert out[-5] == "None", raw + assert out[-4] == "None", raw + + def assert_tcl_tk_library(self, before, activated, deactivated, raw) -> None: + user_values = [os.environ.get("TCL_LIBRARY", "None"), os.environ.get("TK_LIBRARY", "None")] + venv_values = [self._creator.dest.parent / "tcl", self._creator.dest.parent / "tk"] + # activation_python creates these folders only for an environment whose interpreter reports tcl + expected_activated = [str(path) for path in venv_values] if venv_values[0].exists() else user_values + assert (before, activated, deactivated) == (user_values, expected_activated, user_values), raw + + def assert_pkg_config_path(self, before, activated, deactivated, raw) -> None: + user_value = os.environ.get("PKG_CONFIG_PATH") + # comparing entries as paths catches a trailing separator as an extra entry + assert (before, [self.norm_path(entry) for entry in activated.split(os.pathsep)], deactivated) == ( + str(user_value), + [ + self.norm_path(self._creator.dest / "lib" / "pkgconfig"), + *([self.norm_path(user_value)] if user_value else []), + ], + str(user_value), + ), raw def quote(self, s): return self.of_class.quote(s) - def python_cmd(self, cmd): + def python_cmd(self, cmd) -> str: return f"{os.path.basename(sys.executable)} -c {self.quote(cmd)}" def print_python_exe(self): return self.python_cmd("import sys; print(sys.executable)") - def print_os_env_var(self, var): - val = f'"{var}"' - return self.python_cmd(f"import os; import sys; v = os.environ.get({val}); print(v)") + def print_state(self, names: tuple[str, ...]) -> str: + # one launch per phase instead of one per value, since interpreter startup dominates the probe cost + lookups = "; ".join(f'print(os.environ.get("{name}"))' for name in names) + return self.python_cmd(f"import os; import sys; print(sys.executable); {lookups}") def print_prompt(self): return NotImplemented @@ -182,7 +224,7 @@ def norm_path(path): if sys.platform != "win32": result = path else: - from ctypes import create_unicode_buffer, windll # noqa: PLC0415 + from ctypes import create_unicode_buffer, windll # ruff:ignore[import-outside-top-level] buffer_cont = create_unicode_buffer(256) get_long_path_name = windll.kernel32.GetLongPathNameW @@ -192,7 +234,7 @@ def norm_path(path): class RaiseOnNonSourceCall(ActivationTester): - def __init__( # noqa: PLR0913 + def __init__( # ruff:ignore[too-many-arguments] self, of_class, session, @@ -208,11 +250,12 @@ def __call__(self, monkeypatch, tmp_path): env, activate_script = super().__call__(monkeypatch, tmp_path) process = Popen( self.non_source_activate(activate_script), + stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, ) - _out, err_ = process.communicate() + _out, err_ = process.communicate(timeout=60) err = err_.decode("utf-8") assert process.returncode assert self.non_source_fail_message in err @@ -228,21 +271,56 @@ def raise_on_non_source_class(): return RaiseOnNonSourceCall -@pytest.fixture(scope="session", params=[True, False], ids=["with_prompt", "no_prompt"]) -def activation_python(request, tmp_path_factory, special_char_name, current_fastest): +class ActivationCase(NamedTuple): + prompt: bool + tcl: bool + user_env: bool + + +# an L4 orthogonal array: every pair of factors still meets in all four value combinations, at half the cost of the +# full grid; no activation script reads a value that depends on all three factors at once +@pytest.fixture( + scope="session", + params=[ + pytest.param(ActivationCase(prompt=True, tcl=True, user_env=True), id="with_prompt-with_tcl-user_env_set"), + pytest.param(ActivationCase(prompt=True, tcl=False, user_env=False), id="with_prompt-no_tcl-user_env_unset"), + pytest.param(ActivationCase(prompt=False, tcl=True, user_env=False), id="no_prompt-with_tcl-user_env_unset"), + pytest.param(ActivationCase(prompt=False, tcl=False, user_env=True), id="no_prompt-no_tcl-user_env_set"), + ], +) +def activation_case(request: pytest.FixtureRequest) -> ActivationCase: + return request.param + + +@pytest.fixture(scope="session") +def activation_python(activation_case: ActivationCase, tmp_path_factory, special_char_name, current_fastest): dest = os.path.join(str(tmp_path_factory.mktemp("activation-tester-env")), special_char_name) cmd = ["--without-pip", dest, "--creator", current_fastest, "-vv", "--no-periodic-update"] - # `params` is accessed here. https://docs.pytest.org/en/stable/reference/reference.html#pytest-fixture - if request.param: + if activation_case.prompt: cmd += ["--prompt", special_char_name] session = cli_run(cmd) + if activation_case.tcl: + # the interpreter reports tcl_lib only when TCL_LIBRARY is set during its cached probe, so regenerate the scripts + # with the values patched instead + with pytest.MonkeyPatch.context() as monkeypatch: + for name in ("tcl", "tk"): + (path := Path(dest).parent / name).mkdir() + monkeypatch.setattr(session.creator.interpreter, f"{name}_lib", str(path)) + for activator in session.activators: + activator.generate(session.creator) pydoc_test = session.creator.purelib / "pydoc_test.py" pydoc_test.write_text('"""This is pydoc_test.py"""', encoding="utf-8") return session @pytest.fixture -def activation_tester(activation_python, monkeypatch, tmp_path, is_inside_ci): +def activation_tester(activation_case: ActivationCase, activation_python, monkeypatch, tmp_path, is_inside_ci): + for name in ("PKG_CONFIG_PATH", "TCL_LIBRARY", "TK_LIBRARY"): + if activation_case.user_env: + monkeypatch.setenv(name, f"user-{name.lower()}") + else: + monkeypatch.delenv(name, raising=False) + def _tester(tester_class): tester = tester_class(activation_python) if not tester.of_class.supports(activation_python.creator.interpreter): diff --git a/tests/unit/activation/test_activation_support.py b/tests/unit/activation/test_activation_support.py index e25fc96e8..d83213a96 100644 --- a/tests/unit/activation/test_activation_support.py +++ b/tests/unit/activation/test_activation_support.py @@ -1,8 +1,10 @@ from __future__ import annotations from argparse import Namespace +from typing import TYPE_CHECKING, Final import pytest +from python_discovery import PythonInfo from virtualenv.activation import ( BashActivator, @@ -11,45 +13,33 @@ FishActivator, PowerShellActivator, PythonActivator, + XonshActivator, ) -from virtualenv.discovery.py_info import PythonInfo +if TYPE_CHECKING: + from virtualenv.activation.activator import Activator -@pytest.mark.parametrize( - "activator_class", - [BatchActivator, PowerShellActivator, PythonActivator, BashActivator, FishActivator], -) -def test_activator_support_windows(mocker, activator_class): - activator = activator_class(Namespace(prompt=None)) - - interpreter = mocker.Mock(spec=PythonInfo) - interpreter.os = "nt" - assert activator.supports(interpreter) - - -@pytest.mark.parametrize("activator_class", [CShellActivator]) -def test_activator_no_support_windows(mocker, activator_class): - activator = activator_class(Namespace(prompt=None)) - - interpreter = mocker.Mock(spec=PythonInfo) - interpreter.os = "nt" - assert not activator.supports(interpreter) +_SUPPORTED_OS: Final[dict[type[Activator], set[str]]] = { + BashActivator: {"nt", "posix"}, + BatchActivator: {"nt"}, + CShellActivator: {"posix"}, + FishActivator: {"nt", "posix"}, + PowerShellActivator: {"nt", "posix"}, + PythonActivator: {"nt", "posix"}, + XonshActivator: {"nt", "posix"}, +} @pytest.mark.parametrize( - "activator_class", - [BashActivator, CShellActivator, FishActivator, PowerShellActivator, PythonActivator], + ("activator_class", "os_name"), + [ + pytest.param(activator, os_name, id=f"{activator.__name__}-{os_name}") + for activator in _SUPPORTED_OS + for os_name in ("nt", "posix") + ], ) -def test_activator_support_posix(mocker, activator_class): - activator = activator_class(Namespace(prompt=None)) - interpreter = mocker.Mock(spec=PythonInfo) - interpreter.os = "posix" - assert activator.supports(interpreter) - - -@pytest.mark.parametrize("activator_class", [BatchActivator]) -def test_activator_no_support_posix(mocker, activator_class): +def test_activator_support(mocker, activator_class: type[Activator], os_name: str) -> None: activator = activator_class(Namespace(prompt=None)) interpreter = mocker.Mock(spec=PythonInfo) - interpreter.os = "posix" - assert not activator.supports(interpreter) + interpreter.os = os_name + assert activator.supports(interpreter) is (os_name in _SUPPORTED_OS[activator_class]) diff --git a/tests/unit/activation/test_activator.py b/tests/unit/activation/test_activator.py index b6b5b4986..21be96d06 100644 --- a/tests/unit/activation/test_activator.py +++ b/tests/unit/activation/test_activator.py @@ -2,10 +2,13 @@ from argparse import Namespace +import pytest + from virtualenv.activation.activator import Activator -def test_activator_prompt_cwd(monkeypatch, tmp_path): +@pytest.mark.graalpy +def test_activator_prompt_cwd(monkeypatch, tmp_path) -> None: class FakeActivator(Activator): def generate(self, creator): raise NotImplementedError diff --git a/tests/unit/activation/test_bash.py b/tests/unit/activation/test_bash.py index 0a9c588cf..e0f70b98d 100644 --- a/tests/unit/activation/test_bash.py +++ b/tests/unit/activation/test_bash.py @@ -1,11 +1,21 @@ from __future__ import annotations +import shlex +import shutil +import subprocess +import sys from argparse import Namespace +from typing import TYPE_CHECKING, Final import pytest from virtualenv.activation import BashActivator from virtualenv.info import IS_WIN +from virtualenv.run import cli_run + +if TYPE_CHECKING: + from collections.abc import Callable + from pathlib import Path @pytest.mark.skipif(IS_WIN, reason="Github Actions ships with WSL bash") @@ -16,7 +26,7 @@ (None, None, False), ], ) -def test_bash_tkinter_generation(tmp_path, tcl_lib, tk_lib, present): +def test_bash_tkinter_generation(tmp_path, tcl_lib, tk_lib, present) -> None: # GIVEN class MockInterpreter: pass @@ -26,7 +36,7 @@ class MockInterpreter: interpreter.tk_lib = tk_lib class MockCreator: - def __init__(self, dest): + def __init__(self, dest) -> None: self.dest = dest self.bin_dir = dest / "bin" self.bin_dir.mkdir() @@ -46,6 +56,13 @@ def __init__(self, dest): # The teardown logic is always present in deactivate() assert "unset _OLD_VIRTUAL_TCL_LIBRARY" in content assert "unset _OLD_VIRTUAL_TK_LIBRARY" in content + assert "unset _OLD_PKG_CONFIG_PATH" in content + + # PKG_CONFIG_PATH is always set + assert '_OLD_PKG_CONFIG_PATH="${PKG_CONFIG_PATH:-}"' in content + assert 'PKG_CONFIG_PATH="${VIRTUAL_ENV}/lib/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}}"' in content + assert "export PKG_CONFIG_PATH" in content + assert 'PKG_CONFIG_PATH="$_OLD_PKG_CONFIG_PATH"' in content if present: assert 'if [ /path/to/tcl != "" ]; then' in content @@ -63,9 +80,276 @@ def __init__(self, dest): assert "export TCL_LIBRARY" in content +@pytest.fixture +def bash_after_deactivate(tmp_path: Path, current_fastest: str) -> Callable[[str, str], str]: + dest = tmp_path / "venv" + cli_run([ + "--without-pip", + str(dest), + "--creator", + current_fastest, + "--no-periodic-update", + "--activators", + "bash", + ]) + + def probe(setup: str, report: str) -> str: + # activation probes the platform with uname, which an emptied PATH can no longer find + driver = ( + 'uname_path=$(command -v uname)\nuname() { "$uname_path" "$@"; }\n' + f'{setup}\nsource "$1"\ndeactivate\n{report}\n' + ) + return subprocess.run( + ["bash", "--noprofile", "--norc", "-s", "--", str(dest / "bin" / "activate")], + input=driver, + capture_output=True, + text=True, + encoding="utf-8", + timeout=60, + check=False, + ).stdout + + return probe + + +@pytest.mark.skipif(IS_WIN, reason="Github Actions ships with WSL bash") +@pytest.mark.parametrize("original", [pytest.param("", id="empty"), pytest.param("/usr/bin:/bin", id="populated")]) +def test_bash_deactivate_restores_path(bash_after_deactivate: Callable[[str, str], str], original: str) -> None: + out = bash_after_deactivate(f"PATH='{original}'", r'printf "PATH=<%s>\n" "$PATH"') + + assert out == f"PATH=<{original}>\n" + + +@pytest.mark.skipif(IS_WIN, reason="Github Actions ships with WSL bash") +@pytest.mark.parametrize( + ("setup", "expected"), + [ + pytest.param("unset PS1", "unset", id="unset"), + pytest.param("PS1=''", "unset", id="empty"), + pytest.param("PS1='base$ '", "set:base$ ", id="populated"), + ], +) +def test_bash_deactivate_restores_ps1( + bash_after_deactivate: Callable[[str, str], str], setup: str, expected: str +) -> None: + out = bash_after_deactivate(setup, r'printf "%s\n" "${PS1+set:}${PS1-unset}"') + + assert out == f"{expected}\n" + + +@pytest.fixture +def relocated_bash_venv( + tmp_path: Path, current_fastest: str +) -> Callable[[str], tuple[subprocess.CompletedProcess[str], Path, Path]]: + def source_after_move(name: str) -> tuple[subprocess.CompletedProcess[str], Path, Path]: + original = tmp_path / name + cli_run([ + "--without-pip", + str(original), + "--creator", + current_fastest, + "--no-periodic-update", + "--activators", + "bash", + ]) + relocated = tmp_path / "relocated" + shutil.move(original, relocated) + work_dir = tmp_path / "workdir" + work_dir.mkdir() + result = subprocess.run( + ["bash", "-c", f'source "{relocated / "bin" / "activate"}" 2>/dev/null && echo "$VIRTUAL_ENV"'], + capture_output=True, + text=True, + cwd=str(work_dir), + encoding="utf-8", + check=False, + ) + return result, relocated, work_dir + + return source_after_move + + +@pytest.mark.skipif(IS_WIN, reason="Github Actions ships with WSL bash") +@pytest.mark.parametrize( + "name", [pytest.param("original", id="plain"), pytest.param("has(paren)and'quote", id="shell-metacharacters")] +) +def test_bash_activate_relocation_resolves_virtual_env( + relocated_bash_venv: Callable[[str], tuple[subprocess.CompletedProcess[str], Path, Path]], name: str +) -> None: + result, relocated, _ = relocated_bash_venv(name) + + assert result.returncode == 0 + assert result.stdout.strip() == str(relocated) + + +@pytest.mark.skipif(IS_WIN, reason="Github Actions ships with WSL bash") +@pytest.mark.parametrize( + "payload", + [ + pytest.param("x'$(id > PWNED)'y", id="command-substitution"), + pytest.param("x'`id > PWNED`'y", id="backticks"), + pytest.param("a';id > PWNED;'b", id="statement-separator"), + ], +) +def test_bash_activate_relocation_does_not_run_path_commands( + relocated_bash_venv: Callable[[str], tuple[subprocess.CompletedProcess[str], Path, Path]], payload: str +) -> None: + _, _, work_dir = relocated_bash_venv(payload) + + assert not (work_dir / "PWNED").exists() + + +@pytest.mark.skipif(IS_WIN, reason="Github Actions ships with WSL bash") +def test_bash_activate_does_not_export_ps1(tmp_path, current_fastest) -> None: + dest = tmp_path / "env" + cli_run([ + "--without-pip", + str(dest), + "--creator", + current_fastest, + "--no-periodic-update", + "--activators", + "bash", + ]) + activate_script = dest / "bin" / "activate" + print_ps1 = f"{shlex.quote(sys.executable)} -c 'import os; print(os.environ.get(\"PS1\"))'" + result = subprocess.run( + [ + "bash", + "-c", + (f"unset PS1; PS1='base$ '; source \"{activate_script}\" && {print_ps1} && deactivate && {print_ps1}"), + ], + capture_output=True, + encoding="utf-8", + text=True, + ) + assert result.returncode == 0, result.stderr + assert result.stdout.splitlines() == ["None", "None"] + + +@pytest.fixture(params=["bash", "zsh"]) +def prompt_after_activate( + request: pytest.FixtureRequest, tmp_path: Path, current_fastest: str +) -> Callable[[str | None, str], tuple[Path, str]]: + if shutil.which(shell := request.param) is None: + pytest.skip(f"{shell} is not installed") + if shell == "bash": + version: Final[str] = subprocess.run( + ["bash", "-c", 'printf "%s" "$((BASH_VERSINFO[0] * 100 + BASH_VERSINFO[1]))"'], + capture_output=True, + text=True, + check=True, + ).stdout + if int(version) < 404: + pytest.skip("${PS1@P} needs bash 4.4 or later") + + def build(prompt: str | None, env_name: str) -> tuple[Path, str]: + dest: Final[Path] = tmp_path / env_name + args = [ + "--without-pip", + str(dest), + "--creator", + current_fastest, + "--no-periodic-update", + "--activators", + "bash", + ] + if prompt is not None: + args += ["--prompt", prompt] + cli_run(args) + work_dir: Final[Path] = tmp_path / "workdir" + work_dir.mkdir(exist_ok=True) + # the path is passed as $1 so the outer shell cannot expand a payload dir name while locating the script; + # ${PS1@P} renders the prompt exactly as bash does before each interactive command, and in zsh (e) applies + # what PROMPT_SUBST does at each draw before (%) expands the prompt escapes + render: Final[list[str]] = { + "bash": ["bash", "--norc", "--noprofile", "-c", 'source "$1"; printf "%s" "${PS1@P}"'], + "zsh": ["zsh", "-f", "-c", 'setopt prompt_subst; source "$1"; print -rn -- "${(%)${(e)PS1}}"'], + }[shell] + result: Final[subprocess.CompletedProcess[str]] = subprocess.run( + [*render, shell, str(dest / "bin" / "activate")], + capture_output=True, + text=True, + cwd=str(work_dir), + encoding="utf-8", + check=False, + ) + return work_dir, result.stdout + + return build + + +@pytest.mark.skipif(IS_WIN, reason="Github Actions ships with WSL bash") +@pytest.mark.parametrize( + ("prompt", "env_name"), + [ + pytest.param("x$(touch PWNED)y", "env", id="prompt-command-substitution"), + pytest.param("x`touch PWNED`y", "env", id="prompt-backticks"), + pytest.param("x${$(touch PWNED)}y", "env", id="prompt-parameter-expansion"), + pytest.param("x\\$(touch PWNED)y", "env", id="prompt-backslash-command-substitution"), + pytest.param("x\\`touch PWNED`y", "env", id="prompt-backslash-backticks"), + pytest.param(None, "x$(touch PWNED)y", id="dirname-command-substitution"), + pytest.param(None, "A__VIRTUAL_NAME__B/x'$(touch PWNED)'y", id="placeholder-in-parent-dir"), + ], +) +def test_prompt_does_not_run_commands( + prompt_after_activate: Callable[[str | None, str], tuple[Path, str]], prompt: str | None, env_name: str +) -> None: + work_dir, _ = prompt_after_activate(prompt, env_name) + + assert not (work_dir / "PWNED").exists() + + +@pytest.mark.skipif(IS_WIN, reason="Github Actions ships with WSL bash") +@pytest.mark.parametrize( + "prompt", + [ + pytest.param("myenv", id="plain"), + pytest.param("a%n$HOME\\b`c`", id="special-characters"), + pytest.param("a\\$HOME", id="backslash-before-dollar"), + pytest.param("a\\`c`", id="backslash-before-backtick"), + pytest.param("x\\\\y", id="doubled-backslash"), + pytest.param("e\\", id="trailing-backslash"), + ], +) +def test_prompt_renders_name_literally( + prompt_after_activate: Callable[[str | None, str], tuple[Path, str]], prompt: str +) -> None: + _, rendered = prompt_after_activate(prompt, "env") + + assert rendered.startswith(f"({prompt}) ") + + +@pytest.mark.skipif(IS_WIN or shutil.which("dash") is None, reason="needs dash as a POSIX shell") +def test_bash_activate_sources_under_dash(tmp_path: Path, current_fastest: str) -> None: + dest = tmp_path / "env" + cli_run([ + "--without-pip", + str(dest), + "--creator", + current_fastest, + "--no-periodic-update", + "--activators", + "bash", + "--prompt", + "x$y", + ]) + + result = subprocess.run( + ["dash", "-c", '. "$1" && printf "%s" "$VIRTUAL_ENV"', "dash", str(dest / "bin" / "activate")], + capture_output=True, + text=True, + encoding="utf-8", + check=False, + ) + + assert (result.returncode, result.stdout) == (0, str(dest)) + + +@pytest.mark.slow @pytest.mark.skipif(IS_WIN, reason="Github Actions ships with WSL bash") @pytest.mark.parametrize("hashing_enabled", [True, False]) -def test_bash(raise_on_non_source_class, hashing_enabled, activation_tester): +def test_bash(raise_on_non_source_class, hashing_enabled, activation_tester) -> None: class Bash(raise_on_non_source_class): def __init__(self, session) -> None: super().__init__( @@ -83,6 +367,12 @@ def activate_call(self, script): return super().activate_call(script) + " || exit 1" def print_prompt(self): - return self.print_os_env_var("PS1") + # render PS1 the way bash draws it, since the activator escapes the prompt for that expansion; bash before + # 4.4 lacks ${PS1@P}, so undo the escapes once for the prompt-escape decode and once for the expansion + unescape = "s/\\\\\\([$`\\\\]\\)/\\1/g" + return ( + 'if ((BASH_VERSINFO[0] * 100 + BASH_VERSINFO[1] >= 404)); then printf "%s\\n" "${PS1@P}"; ' + f"else printf '%s\\n' \"$PS1\" | sed -e '{unescape}' -e '{unescape}'; fi" + ) activation_tester(Bash) diff --git a/tests/unit/activation/test_batch.py b/tests/unit/activation/test_batch.py index db2595860..3f85db4ed 100644 --- a/tests/unit/activation/test_batch.py +++ b/tests/unit/activation/test_batch.py @@ -1,10 +1,74 @@ from __future__ import annotations +import json +import os +import subprocess +import sys from argparse import Namespace +from typing import TYPE_CHECKING, Final import pytest +from virtualenv import session_via_cli from virtualenv.activation import BatchActivator +from virtualenv.config.cli.parser import VirtualEnvOptions +from virtualenv.info import IS_WIN + +if TYPE_CHECKING: + from pathlib import Path + + from virtualenv.create.creator import Creator + + +@pytest.fixture(scope="session") +def special_char_name(): + """Exclude ampersands, which batch activation does not support in paths.""" + base = "'\";e-$ èрт🚒♞中片-j" + if IS_WIN: + base = base.replace('"', "").replace(";", "") + encoding = "ascii" if IS_WIN else sys.getfilesystemencoding() + result = "" + for char in base: + try: + trip = char.encode(encoding, errors="strict").decode(encoding) + if char == trip: + result += char + except ValueError: # ruff:ignore[try-except-in-loop] + continue + assert result + return result + + +def test_batch_pydoc_bat_quoting(tmp_path) -> None: + """Test that pydoc.bat properly quotes python.exe path to handle spaces.""" + + # GIVEN: A mock interpreter + class MockInterpreter: + os = "nt" + tcl_lib = None + tk_lib = None + + class MockCreator: + def __init__(self, dest) -> None: + self.dest = dest + self.bin_dir = dest / "Scripts" + self.bin_dir.mkdir(parents=True) + self.interpreter = MockInterpreter() + self.pyenv_cfg = {} + self.env_name = "test-env" + + creator = MockCreator(tmp_path) + options = Namespace(prompt=None) + activator = BatchActivator(options) + + # WHEN: Generate activation scripts + activator.generate(creator) + + # THEN: pydoc.bat should quote python.exe to handle paths with spaces + pydoc_content = (creator.bin_dir / "pydoc.bat").read_text(encoding="utf-8") + + # The python.exe should be quoted to handle paths with spaces like "C:\Program Files\Python39\python.exe" + assert '"python.exe"' in pydoc_content, f"python.exe should be quoted in pydoc.bat. Content:\n{pydoc_content}" @pytest.mark.parametrize( @@ -14,7 +78,7 @@ (None, None, False), ], ) -def test_batch_tkinter_generation(tmp_path, tcl_lib, tk_lib, present): +def test_batch_tkinter_generation(tmp_path, tcl_lib, tk_lib, present) -> None: # GIVEN class MockInterpreter: os = "nt" @@ -24,7 +88,7 @@ class MockInterpreter: interpreter.tk_lib = tk_lib class MockCreator: - def __init__(self, dest): + def __init__(self, dest) -> None: self.dest = dest self.bin_dir = dest / "bin" self.bin_dir.mkdir() @@ -42,6 +106,13 @@ def __init__(self, dest): deactivate_content = (creator.bin_dir / "deactivate.bat").read_text(encoding="utf-8") # THEN + # PKG_CONFIG_PATH is always set + assert '@if defined PKG_CONFIG_PATH @set "_OLD_PKG_CONFIG_PATH=%PKG_CONFIG_PATH%"' in activate_content + assert '@set "PKG_CONFIG_PATH=%VIRTUAL_ENV%\\lib\\pkgconfig;%PKG_CONFIG_PATH%"' in activate_content + assert '@if defined _OLD_PKG_CONFIG_PATH @set "PKG_CONFIG_PATH=%_OLD_PKG_CONFIG_PATH%"' in deactivate_content + assert "@if not defined _OLD_PKG_CONFIG_PATH @set PKG_CONFIG_PATH=" in deactivate_content + assert "@set _OLD_PKG_CONFIG_PATH=" in deactivate_content + if present: assert '@if NOT "C:\\tcl"=="" @set "TCL_LIBRARY=C:\\tcl"' in activate_content assert '@if NOT "C:\\tk"=="" @set "TK_LIBRARY=C:\\tk"' in activate_content @@ -52,8 +123,81 @@ def __init__(self, dest): assert '@if NOT ""=="" @set "TK_LIBRARY="' in activate_content -@pytest.mark.usefixtures("activation_python") -def test_batch(activation_tester_class, activation_tester, tmp_path): +# every field feeds the same character check, so the tcl and tk fields need only one of the characters +@pytest.mark.parametrize( + ("field", "character"), + [ + pytest.param("dest", "&", id="destination-ampersand"), + pytest.param("dest", "^", id="destination-caret"), + pytest.param("dest", "!", id="destination-exclamation"), + pytest.param("tcl_lib", "&", id="tcl-ampersand"), + pytest.param("tk_lib", "&", id="tk-ampersand"), + ], +) +def test_batch_skips_changed_paths( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, character: str, field: str +) -> None: + creator: Final[Creator] = session_via_cli( + [str(tmp_path / (f"env{character}name" if field == "dest" else "env")), "--no-seed"], + setup_logging=False, + ).creator + if field != "dest": + monkeypatch.setattr(creator.interpreter, field, str(tmp_path / f"lib{character}name"), raising=False) + creator.bin_dir.mkdir(parents=True) + assert BatchActivator(VirtualEnvOptions(prompt=None)).generate(creator) == [] + assert list(creator.bin_dir.iterdir()) == [] + assert "skipping batch activation scripts" in caplog.text + assert "cannot preserve" in caplog.text + + +@pytest.mark.skipif(not IS_WIN, reason="requires cmd.exe") +@pytest.mark.parametrize( + "name", + [ + pytest.param("env space", id="space"), + pytest.param("env(x86)", id="parentheses"), + pytest.param("env%SECRET%", id="percent"), + ], +) +@pytest.mark.parametrize("delayed", [pytest.param("OFF", id="normal"), pytest.param("ON", id="delayed-expansion")]) +def test_batch_path_round_trip(tmp_path: Path, name: str, delayed: str) -> None: + creator: Final[Creator] = session_via_cli([str(tmp_path / name), "--no-seed"], setup_logging=False).creator + creator.bin_dir.mkdir(parents=True) + BatchActivator(VirtualEnvOptions(prompt="roundtrip")).generate(creator) + snapshot: Final[str] = ( + f'@"{sys.executable}" -I -c "import json,os; ' + "print(json.dumps({k:os.environ.get(k) for k in ('VIRTUAL_ENV','PATH','PROMPT')}))\"" + ) + (creator.bin_dir / "check.bat").write_text( + '@echo off\n@set VIRTUAL_ENV=\n@set _OLD_VIRTUAL_PATH=\n@set _OLD_VIRTUAL_PROMPT=\n@set "PROMPT=original"\n' + f"@call activate.bat\n{snapshot}\n@call activate.bat\n{snapshot}\n@call deactivate.bat\n{snapshot}", + encoding="utf-8", + ) + result: Final[subprocess.CompletedProcess[str]] = subprocess.run( + [os.environ["COMSPEC"], "/D", f"/V:{delayed}", "/C", "check.bat"], + cwd=creator.bin_dir, + capture_output=True, + text=True, + encoding="utf-8", + check=True, + env={**os.environ, "SECRET": "expanded", "VIRTUAL_ENV_DISABLE_PROMPT": ""}, + ) + active: Final[dict[str, str]] = { + "VIRTUAL_ENV": str(creator.dest), + "PATH": f"{creator.bin_dir};{os.environ['PATH']}", + "PROMPT": "(roundtrip) original", + } + assert [json.loads(line) for line in result.stdout.splitlines()] == [ + active, + active, + {"VIRTUAL_ENV": None, "PATH": os.environ["PATH"], "PROMPT": "original"}, + ] + + +@pytest.mark.parametrize("activations", [1, 2], ids=["activate_once", "activate_twice"]) +def test_batch(activation_python, activation_tester_class, activation_tester, tmp_path, activations) -> None: + if not (activation_python.creator.bin_dir / "activate.bat").exists(): + pytest.skip("Batch activation does not support this destination") version_script = tmp_path / "version.bat" version_script.write_text("ver", encoding="utf-8") @@ -70,20 +214,25 @@ def __init__(self, session) -> None: def _get_test_lines(self, activate_script): return ["@echo off", *super()._get_test_lines(activate_script)] + def activate_call(self, script): + # activating again without deactivating must still restore the values from before the first activation + return " & ".join([super().activate_call(script)] * activations) + def quote(self, s): if '"' in s or " " in s: text = s.replace('"', r"\"") return f'"{text}"' return s - def print_prompt(self): + def print_prompt(self) -> str: return 'echo "%PROMPT%"' activation_tester(Batch) -@pytest.mark.usefixtures("activation_python") -def test_batch_output(activation_tester_class, activation_tester, tmp_path): +def test_batch_output(activation_python, activation_tester_class, activation_tester, tmp_path) -> None: + if not (activation_python.creator.bin_dir / "activate.bat").exists(): + pytest.skip("Batch activation does not support this destination") version_script = tmp_path / "version.bat" version_script.write_text("ver", encoding="utf-8") @@ -98,11 +247,10 @@ def __init__(self, session) -> None: self.unix_line_ending = False def _get_test_lines(self, activate_script): - """ - Build intermediary script which will be then called. - In the script just activate environment, call echo to get current - echo setting, and then deactivate. This ensures that echo setting - is preserved and no unwanted output appears. + """Build an intermediary script that activates, echoes the current echo setting, then deactivates. + + This is what proves echo state survives activation and deactivation without leaking unwanted output. + """ intermediary_script_path = str(tmp_path / "intermediary.bat") activate_script_quoted = self.quote(str(activate_script)) @@ -114,7 +262,7 @@ def _get_test_lines(self, activate_script): f"@call {intermediary_script_path}", ] - def assert_output(self, out, raw, tmp_path): # noqa: ARG002 + def assert_output(self, out, raw, tmp_path) -> None: # ruff:ignore[unused-method-argument] assert out[0] == "ECHO is on.", raw def quote(self, s): @@ -123,7 +271,7 @@ def quote(self, s): return f'"{text}"' return s - def print_prompt(self): + def print_prompt(self) -> str: return 'echo "%PROMPT%"' activation_tester(Batch) diff --git a/tests/unit/activation/test_csh.py b/tests/unit/activation/test_csh.py index 5cea684ec..da0e4a32f 100644 --- a/tests/unit/activation/test_csh.py +++ b/tests/unit/activation/test_csh.py @@ -3,12 +3,21 @@ import sys from argparse import Namespace from shutil import which -from subprocess import check_output +from subprocess import check_output, run +from typing import TYPE_CHECKING import pytest from packaging.version import Version from virtualenv.activation import CShellActivator +from virtualenv.info import IS_WIN +from virtualenv.run import cli_run + +if TYPE_CHECKING: + from collections.abc import Callable + from pathlib import Path + +TCSH = which("tcsh") @pytest.mark.parametrize( @@ -18,7 +27,7 @@ (None, None, False), ], ) -def test_cshell_tkinter_generation(tmp_path, tcl_lib, tk_lib, present): +def test_cshell_tkinter_generation(tmp_path, tcl_lib, tk_lib, present) -> None: # GIVEN class MockInterpreter: pass @@ -28,7 +37,7 @@ class MockInterpreter: interpreter.tk_lib = tk_lib class MockCreator: - def __init__(self, dest): + def __init__(self, dest) -> None: self.dest = dest self.bin_dir = dest / "bin" self.bin_dir.mkdir() @@ -42,18 +51,165 @@ def __init__(self, dest): # WHEN activator.generate(creator) - content = (creator.bin_dir / "activate.csh").read_text(encoding="utf-8") + activate = (creator.bin_dir / "activate.csh").read_text(encoding="utf-8") + deactivate = (creator.bin_dir / "deactivate.csh").read_text(encoding="utf-8") + + # the restore side carries no per-venv placeholders, so it is identical regardless of `present` + assert "if ($?_OLD_PKG_CONFIG_PATH) then" in deactivate + assert 'setenv PKG_CONFIG_PATH "$_OLD_PKG_CONFIG_PATH:q"' in deactivate + assert "unset _OLD_PKG_CONFIG_PATH" in deactivate + assert "if ($?_OLD_VIRTUAL_TCL_LIBRARY) then" in deactivate + assert "if ($?_OLD_VIRTUAL_TK_LIBRARY) then" in deactivate + + # PKG_CONFIG_PATH is always saved on activation + assert 'set _OLD_PKG_CONFIG_PATH="$PKG_CONFIG_PATH"' in activate + assert 'setenv PKG_CONFIG_PATH "${VIRTUAL_ENV}/lib/pkgconfig:${PKG_CONFIG_PATH}"' in activate + assert 'setenv PKG_CONFIG_PATH "${VIRTUAL_ENV}/lib/pkgconfig"' in activate if present: - assert "test $?_OLD_VIRTUAL_TCL_LIBRARY != 0" in content - assert "test $?_OLD_VIRTUAL_TK_LIBRARY != 0" in content - assert "setenv TCL_LIBRARY /path/to/tcl" in content - assert "setenv TK_LIBRARY /path/to/tk" in content + assert "setenv TCL_LIBRARY /path/to/tcl" in activate + assert "setenv TK_LIBRARY /path/to/tk" in activate else: - assert "setenv TCL_LIBRARY ''" in content + assert "setenv TCL_LIBRARY ''" in activate + + +def test_cshell_generates_deactivate_script(tmp_path) -> None: + class MockCreator: + def __init__(self, dest) -> None: + self.dest = dest + self.bin_dir = dest / "bin" + self.bin_dir.mkdir() + self.interpreter = type("MockInterpreter", (), {"tcl_lib": None, "tk_lib": None})() + self.pyenv_cfg = {} + self.env_name = "my-env" + + creator = MockCreator(tmp_path) + CShellActivator(Namespace(prompt=None)).generate(creator) + + assert (creator.bin_dir / "deactivate.csh").exists() + assert "deactivate.csh" in (creator.bin_dir / "activate.csh").read_text(encoding="utf-8") + + +def _create_csh_venv(dest: Path, current_fastest: str, prompt: str | None = None) -> str: + cmd = ["--without-pip", str(dest), "--creator", current_fastest, "--no-periodic-update", "--activators", "cshell"] + if prompt is not None: + cmd += ["--prompt", prompt] + cli_run(cmd) + return (dest / "bin" / "activate.csh").read_text(encoding="utf-8") + + +@pytest.fixture +def csh_venv(tmp_path: Path, current_fastest: str) -> Callable[..., tuple[Path, str]]: + def create(name: str, *, prompt: str | None = None) -> tuple[Path, str]: + (dest := tmp_path / name / "venv").parent.mkdir(parents=True) + return dest, _create_csh_venv(dest, current_fastest, prompt) + + return create + + +# the deactivate tests only source the script, so they share one environment +@pytest.fixture(scope="module") +def csh_plain_activate(tmp_path_factory: pytest.TempPathFactory, current_fastest: str) -> Path: + _create_csh_venv(dest := tmp_path_factory.mktemp("plain") / "venv", current_fastest) + return dest / "bin" / "activate.csh" + + +@pytest.mark.skipif(IS_WIN, reason="csh is not supported on Windows") +def test_cshell_escapes_history_character(csh_venv: Callable[..., tuple[Path, str]]) -> None: + dest, content = csh_venv("has!bang") + + assert f"setenv VIRTUAL_ENV '{str(dest).replace('!', chr(92) + '!')}'\n" in content + + +@pytest.mark.skipif(IS_WIN, reason="csh is not supported on Windows") +@pytest.mark.parametrize( + ("prompt", "tcsh_escaped", "plain_escaped"), + [ + pytest.param("plain", "plain", "plain", id="plain"), + pytest.param("has!bang", "has\\\\!bang", "has\\\\!bang", id="bang"), + pytest.param("has%pct", "has%%pct", "has%pct", id="percent"), + pytest.param("a!b%p c", "a\\\\!b%%p c", "a\\\\!b%p c", id="bang-and-percent"), + ], +) +def test_cshell_escapes_prompt_expansion( + csh_venv: Callable[..., tuple[Path, str]], prompt: str, tcsh_escaped: str, plain_escaped: str +) -> None: + _, content = csh_venv(prompt, prompt=prompt) + + # tcsh treats a bare % as the start of a prompt escape and needs it doubled; plain csh has no such escape, and + # doubling it there would show two literal percent signs, so activate.csh picks the branch to use at runtime + assert f"set prompt = '(''{tcsh_escaped}'') '" in content + assert f"set prompt = '(''{plain_escaped}'') '" in content + + +@pytest.mark.skipif(IS_WIN, reason="csh is not supported on Windows") +@pytest.mark.skipif(TCSH is None, reason="tcsh is not installed") +@pytest.mark.parametrize( + "name", + [ + pytest.param("plain", id="plain"), + pytest.param("has!bang", id="bang"), + pytest.param("has!!doublebang", id="double-bang"), + pytest.param("has'quote!bang", id="quote-and-bang"), + pytest.param("A__VIRTUAL_PROMPT_DISPLAY_TCSH__B", id="display-placeholder"), + ], +) +def test_cshell_activates_path_with_special_characters( + csh_venv: Callable[[str], tuple[Path, str]], tmp_path: Path, name: str +) -> None: + dest, content = csh_venv(name) + # source from an ASCII path so the driver's own quoting cannot stand in for the script's + script = tmp_path / "activate.csh" + script.write_text(content, encoding="utf-8") + + result = run( + [TCSH, "-c", f'source {script} && printf "%s" "$VIRTUAL_ENV"'], + capture_output=True, + text=True, + encoding="utf-8", + timeout=90, + check=False, + ) + + assert result.stdout == str(dest), result.stderr + + +@pytest.mark.skipif(IS_WIN, reason="csh is not supported on Windows") +@pytest.mark.skipif(TCSH is None, reason="tcsh is not installed") +@pytest.mark.parametrize("original", [pytest.param("", id="empty"), pytest.param("/usr/bin:/bin", id="populated")]) +def test_cshell_deactivate_restores_path(csh_plain_activate: Path, original: str) -> None: + driver = f'setenv PATH "{original}"\nsource {csh_plain_activate}\ndeactivate\necho "PATH=<$PATH>"\n' + + result = run([TCSH, "-f"], input=driver, capture_output=True, text=True, encoding="utf-8", timeout=90, check=False) + + assert result.stdout == f"PATH=<{original}>\n", result.stderr + + +@pytest.mark.skipif(IS_WIN, reason="csh is not supported on Windows") +@pytest.mark.skipif(TCSH is None, reason="tcsh is not installed") +def test_cshell_deactivate_removes_aliases(csh_plain_activate: Path) -> None: + driver = f'source {csh_plain_activate}\ndeactivate\nalias deactivate\nalias pydoc\necho "done"\n' + + result = run([TCSH, "-f"], input=driver, capture_output=True, text=True, encoding="utf-8", timeout=90, check=False) + + assert result.stdout == "done\n", result.stderr + + +@pytest.mark.skipif(IS_WIN, reason="csh is not supported on Windows") +@pytest.mark.skipif(TCSH is None, reason="tcsh is not installed") +def test_cshell_reactivation_restores_original_path(csh_plain_activate: Path) -> None: + driver = ( + f'setenv PATH "/original/path"\nsource {csh_plain_activate}\nsource {csh_plain_activate}\ndeactivate\n' + 'echo "PATH=<$PATH>"\n' + ) + + result = run([TCSH, "-f"], input=driver, capture_output=True, text=True, encoding="utf-8", timeout=90, check=False) + + assert result.stdout == "PATH=\n", result.stderr -def test_csh(activation_tester_class, activation_tester): +@pytest.mark.slow +def test_csh(activation_tester_class, activation_tester) -> None: exe = f"tcsh{'.exe' if sys.platform == 'win32' else ''}" if which(exe): version_text = check_output([exe, "--version"], text=True, encoding="utf-8") @@ -65,7 +221,7 @@ class Csh(activation_tester_class): def __init__(self, session) -> None: super().__init__(CShellActivator, session, "csh", "activate.csh", "csh") - def print_prompt(self): + def print_prompt(self) -> str: # Original csh doesn't print the last newline, # breaking the test; hence the trailing echo. return "echo 'source \"$VIRTUAL_ENV/bin/activate.csh\"; echo $prompt' | csh -i ; echo" diff --git a/tests/unit/activation/test_fish.py b/tests/unit/activation/test_fish.py index c15a8f513..95d1bb073 100644 --- a/tests/unit/activation/test_fish.py +++ b/tests/unit/activation/test_fish.py @@ -1,59 +1,182 @@ from __future__ import annotations import os +import shutil +import subprocess import sys from argparse import Namespace +from types import SimpleNamespace +from typing import TYPE_CHECKING import pytest from virtualenv.activation import FishActivator from virtualenv.info import IS_WIN +from virtualenv.run import cli_run + +if TYPE_CHECKING: + from collections.abc import Callable + from pathlib import Path + +FISH = shutil.which("fish") + + +@pytest.fixture +def rendered_activate_fish(tmp_path: Path) -> Callable[[str | None, str | None], Path]: + def render(tcl_lib: str | None, tk_lib: str | None) -> Path: + creator = SimpleNamespace( + dest=tmp_path, + bin_dir=tmp_path / "bin", + interpreter=SimpleNamespace(tcl_lib=tcl_lib, tk_lib=tk_lib), + pyenv_cfg={}, + env_name="my-env", + ) + creator.bin_dir.mkdir() + FishActivator(Namespace(prompt=None)).generate(creator) + return creator.bin_dir / "activate.fish" + + return render @pytest.mark.parametrize( ("tcl_lib", "tk_lib", "present"), [ ("/path/to/tcl", "/path/to/tk", True), + ("/Program Files/tcl", "/Program Files/tk", True), (None, None, False), ], ) -def test_fish_tkinter_generation(tmp_path, tcl_lib, tk_lib, present): - # GIVEN - class MockInterpreter: - pass - - interpreter = MockInterpreter() - interpreter.tcl_lib = tcl_lib - interpreter.tk_lib = tk_lib - - class MockCreator: - def __init__(self, dest): - self.dest = dest - self.bin_dir = dest / "bin" - self.bin_dir.mkdir() - self.interpreter = interpreter - self.pyenv_cfg = {} - self.env_name = "my-env" - - creator = MockCreator(tmp_path) - options = Namespace(prompt=None) - activator = FishActivator(options) - - # WHEN - activator.generate(creator) - content = (creator.bin_dir / "activate.fish").read_text(encoding="utf-8") - - # THEN +def test_fish_tkinter_generation( + rendered_activate_fish: Callable[[str | None, str | None], Path], + tcl_lib: str | None, + tk_lib: str | None, + present: bool, +) -> None: + content = rendered_activate_fish(tcl_lib, tk_lib).read_text(encoding="utf-8") + + assert 'set -gx _OLD_PKG_CONFIG_PATH "$PKG_CONFIG_PATH"' in content + assert 'set -gx PKG_CONFIG_PATH "$VIRTUAL_ENV/lib/pkgconfig:$PKG_CONFIG_PATH"' in content + assert "set -e _OLD_PKG_CONFIG_PATH" in content + if present: - assert "set -gx TCL_LIBRARY '/path/to/tcl'" in content - assert "set -gx TK_LIBRARY '/path/to/tk'" in content + assert f"set -gx TCL_LIBRARY {FishActivator.quote(tcl_lib)}\n" in content + assert f"set -gx TK_LIBRARY {FishActivator.quote(tk_lib)}\n" in content else: - assert "if test -n ''\n if set -q TCL_LIBRARY;" in content - assert "if test -n ''\n if set -q TK_LIBRARY;" in content + assert "if test -n ''\n set -gx _OLD_VIRTUAL_TCL_LIBRARY" in content + assert "if test -n ''\n set -gx _OLD_VIRTUAL_TK_LIBRARY" in content + + +@pytest.mark.skipif(IS_WIN, reason="fish is not available on Windows") +@pytest.mark.skipif(FISH is None, reason="fish is not installed") +def test_fish_tkinter_path_does_not_run_commands( + rendered_activate_fish: Callable[[str | None, str | None], Path], tmp_path: Path +) -> None: + marker = tmp_path / "PWNED" + script = rendered_activate_fish(f"/tcl/(touch {marker})/lib", "/tk/lib") + + subprocess.run([FISH, "-c", f"source '{script}'"], capture_output=True, text=True, timeout=60, check=False) + assert not marker.exists() + +@pytest.mark.skipif(IS_WIN, reason="fish is not available on Windows") +@pytest.mark.skipif(FISH is None, reason="fish is not installed") +@pytest.mark.parametrize( + ("prompt", "env_name"), + [ + pytest.param(r"x\'$(touch PWNED)'\'", "env", id="prompt-backslash-quote"), + pytest.param(None, r"x\'$(touch PWNED)'\'", id="dirname-backslash-quote"), + ], +) +def test_fish_activate_does_not_run_commands( + tmp_path: Path, current_fastest: str, prompt: str | None, env_name: str +) -> None: + dest = tmp_path / env_name + args = ["--without-pip", str(dest), "--creator", current_fastest, "--no-periodic-update", "--activators", "fish"] + if prompt is not None: + args += ["--prompt", prompt] + cli_run(args) + (work_dir := tmp_path / "workdir").mkdir() + + # the path is passed as an argument so the driver cannot expand a payload dir name while locating the script + result = subprocess.run( + [FISH, "--no-config", "-c", 'source $argv[1]; printf "%s" "$VIRTUAL_ENV"', str(dest / "bin" / "activate.fish")], + capture_output=True, + text=True, + encoding="utf-8", + cwd=str(work_dir), + timeout=60, + check=False, + ) + + assert not (work_dir / "PWNED").exists() + assert result.stdout == str(dest), result.stderr + + +@pytest.mark.skipif(IS_WIN, reason="fish is not available on Windows") +@pytest.mark.skipif(FISH is None, reason="fish is not installed") +@pytest.mark.parametrize( + ("original", "expected"), + [ + pytest.param("", "", id="empty"), + # fish joins a path variable with colons when it is expanded inside quotes + pytest.param("/usr/bin /bin", "/usr/bin:/bin", id="populated"), + ], +) +def test_fish_deactivate_restores_path(tmp_path, current_fastest, original, expected) -> None: + dest = tmp_path / "venv" + cli_run([ + "--without-pip", + str(dest), + "--creator", + current_fastest, + "--no-periodic-update", + "--activators", + "fish", + ]) + + out = subprocess.run( + [ + FISH, + "--no-config", + "-c", + f"set -gx PATH {original}\nsource '{dest / 'bin' / 'activate.fish'}'\ndeactivate\necho \"PATH=<$PATH>\"\n", + ], + capture_output=True, + text=True, + encoding="utf-8", + timeout=60, + check=False, + ).stdout + + assert out == f"PATH=<{expected}>\n" + + +@pytest.mark.skipif(IS_WIN, reason="fish is not available on Windows") +@pytest.mark.skipif(FISH is None, reason="fish is not installed") +def test_fish_prompt_survives_shadowed_source(activation_python, tmp_path) -> None: + # A user function shadowing `.`/`source` must not hijack the prompt's exit-status restore. Source a copy of the + # rendered script from an ASCII path so the driver avoids the venv path's special characters. + script = tmp_path / "activate.fish" + rendered = activation_python.creator.bin_dir / "activate.fish" + script.write_text(rendered.read_text(encoding="utf-8"), encoding="utf-8") + start, trap = tmp_path / "start", tmp_path / "trap" + start.mkdir() + trap.mkdir() + driver = tmp_path / "driver.fish" + driver.write_text( + f"cd '{start}'\nfunction . ; cd '{trap}' ; end\nsource '{script}'\nfish_prompt\necho \"PWD=$PWD\"\n", + encoding="utf-8", + ) + out = subprocess.run( + [FISH, str(driver)], capture_output=True, text=True, encoding="utf-8", timeout=60, check=True + ).stdout + assert f"PWD={start}\n" in out, out + + +@pytest.mark.slow @pytest.mark.skipif(IS_WIN, reason="we have not setup fish in CI yet") -def test_fish(activation_tester_class, activation_tester, monkeypatch, tmp_path): +def test_fish(activation_tester_class, activation_tester, monkeypatch, tmp_path) -> None: monkeypatch.setenv("HOME", str(tmp_path)) fish_conf_dir = tmp_path / ".config" / "fish" fish_conf_dir.mkdir(parents=True) @@ -63,57 +186,51 @@ class Fish(activation_tester_class): def __init__(self, session) -> None: super().__init__(FishActivator, session, "fish", "activate.fish", "fish") - def print_prompt(self): + def print_prompt(self) -> str: return "fish_prompt" def _get_test_lines(self, activate_script): + names = ("VIRTUAL_ENV", "VIRTUAL_ENV_PROMPT", "PATH", "TCL_LIBRARY", "TK_LIBRARY", "PKG_CONFIG_PATH") return [ - self.print_python_exe(), - self.print_os_env_var("VIRTUAL_ENV"), - self.print_os_env_var("VIRTUAL_ENV_PROMPT"), - self.print_os_env_var("PATH"), + self.print_state(names), self.activate_call(activate_script), - self.print_python_exe(), - self.print_os_env_var("VIRTUAL_ENV"), - self.print_os_env_var("VIRTUAL_ENV_PROMPT"), - self.print_os_env_var("PATH"), + self.print_state(names), self.print_prompt(), # \\ loads documentation from the virtualenv site packages self.pydoc_call, self.deactivate, - self.print_python_exe(), - self.print_os_env_var("VIRTUAL_ENV"), - self.print_os_env_var("VIRTUAL_ENV_PROMPT"), - self.print_os_env_var("PATH"), + self.print_state(names), "", # just finish with an empty new line ] - def assert_output(self, out, raw, _): + def assert_output(self, out, raw, _) -> None: """Compare _get_test_lines() with the expected values.""" assert out[0], raw assert out[1] == "None", raw assert out[2] == "None", raw + self.assert_tcl_tk_library(out[4:6], out[11:13], out[-3:-1], raw) + self.assert_pkg_config_path(out[6], out[13], out[-1], raw) # self.activate_call(activate_script) runs at this point expected = self._creator.exe.parent / os.path.basename(sys.executable) - assert self.norm_path(out[4]) == self.norm_path(expected), raw - assert self.norm_path(out[5]) == self.norm_path(self._creator.dest).replace("\\\\", "\\"), raw - assert out[6] == self._creator.env_name + assert self.norm_path(out[7]) == self.norm_path(expected), raw + assert self.norm_path(out[8]) == self.norm_path(self._creator.dest).replace("\\\\", "\\"), raw + assert out[9] == self._creator.env_name # Some attempts to test the prompt output print more than 1 line. # So we need to check if the prompt exists on any of them. prompt_text = f"({self._creator.env_name}) " - assert any(prompt_text in line for line in out[7:-5]), raw + assert any(prompt_text in line for line in out[14:-8]), raw - assert out[-5] == "wrote pydoc_test.html", raw + assert out[-8] == "wrote pydoc_test.html", raw content = tmp_path / "pydoc_test.html" assert content.exists(), raw # post deactivation, same as before - assert out[-4] == out[0], raw - assert out[-3] == "None", raw - assert out[-2] == "None", raw + assert out[-7] == out[0], raw + assert out[-6] == "None", raw + assert out[-5] == "None", raw # Check that the PATH is restored - assert out[3] == out[13], raw + assert out[3] == out[-4], raw # Check that PATH changed after activation - assert out[3] != out[8], raw + assert out[3] != out[10], raw activation_tester(Fish) diff --git a/tests/unit/activation/test_nushell.py b/tests/unit/activation/test_nushell.py index 08c5cb1a1..75584ec21 100644 --- a/tests/unit/activation/test_nushell.py +++ b/tests/unit/activation/test_nushell.py @@ -1,13 +1,21 @@ from __future__ import annotations +import subprocess from argparse import Namespace from shutil import which +from typing import TYPE_CHECKING + +import pytest from virtualenv.activation import NushellActivator from virtualenv.info import IS_WIN +from virtualenv.run import cli_run + +if TYPE_CHECKING: + from pathlib import Path -def test_nushell_tkinter_generation(tmp_path): +def test_nushell_tkinter_generation(tmp_path) -> None: # GIVEN class MockInterpreter: pass @@ -19,7 +27,7 @@ class MockInterpreter: quoted_tk_path = NushellActivator.quote(interpreter.tk_lib) class MockCreator: - def __init__(self, dest): + def __init__(self, dest) -> None: self.dest = dest self.bin_dir = dest / "bin" self.bin_dir.mkdir() @@ -36,28 +44,44 @@ def __init__(self, dest): content = (creator.bin_dir / "activate.nu").read_text(encoding="utf-8") # THEN + # PKG_CONFIG_PATH is always set + assert "let old_pkg_config_path = if (has-env 'PKG_CONFIG_PATH')" in content + assert "let new_pkg_config_path = " in content + assert "PKG_CONFIG_PATH: $new_pkg_config_path" in content + expected_tcl = f"let $new_env = $new_env | insert TCL_LIBRARY {quoted_tcl_path}" expected_tk = f"let $new_env = $new_env | insert TK_LIBRARY {quoted_tk_path}" assert expected_tcl in content assert expected_tk in content + # overlay hide is a parser keyword: a def body would fail at parse time because the overlay doesn't exist yet + # when the def is compiled. The alias defers that check to call time, when the overlay is active. + assert "export alias deactivate = overlay hide activate" in content + # nushell shows one line of context before the error site, so placing the hint comment directly above the alias + # makes it appear in the error output users see when they activate via `use *` or a custom name (gh-3103). + lines = content.splitlines() + alias_idx = next(i for i, line in enumerate(lines) if "export alias deactivate" in line) + assert alias_idx > 0 + assert "overlay use activate.nu" in lines[alias_idx - 1] + -def test_nushell(activation_tester_class, activation_tester): +@pytest.mark.slow +def test_nushell(activation_tester_class, activation_tester) -> None: class Nushell(activation_tester_class): def __init__(self, session) -> None: - cmd = which("nu") - if cmd is None and IS_WIN: - cmd = "c:\\program files\\nu\\bin\\nu.exe" - - super().__init__(NushellActivator, session, cmd, "activate.nu", "nu") + super().__init__(NushellActivator, session, which("nu"), "activate.nu", "nu") self.activate_cmd = "overlay use" self.unix_line_ending = not IS_WIN - def print_prompt(self): + def print_prompt(self) -> str: return r"print $env.VIRTUAL_PREFIX" + def assert_tcl_tk_library(self, before, activated, deactivated, raw) -> None: + # activate.nu never sets TCL_LIBRARY or TK_LIBRARY: its `let $new_env` inside `if` only shadows the variable + assert before == activated == deactivated, raw + def activate_call(self, script): # Commands are called without quotes in Nushell cmd = self.activate_cmd @@ -65,3 +89,35 @@ def activate_call(self, script): return f"{cmd} {scr}".strip() activation_tester(Nushell) + + +def test_nushell_deactivate_errors(tmp_path: Path) -> None: + """Regression for gh-3103: both misuse patterns give actionable inline errors. + + `^nu | complete` captures stderr from child invocations without aborting the outer script, so both cases run in a + single subprocess call. + + """ + nu = which("nu") + if nu is None: + pytest.skip("nu not installed") + + activate_nu = cli_run(["--without-pip", str(tmp_path / "venv")]).creator.bin_dir / "activate.nu" + quoted = NushellActivator.quote(str(activate_nu)) + + # `to nuon` re-quotes the path so it remains valid nushell syntax after string interpolation. + script = f"""\ +let path = {quoted} +let r1 = (^nu --commands $"use ($path | to nuon) *; deactivate" | complete) +if ($r1.exit_code == 0) {{ error make {{ msg: "expected deactivate to fail for use-star" }} }} +if not ("not an active overlay" in $r1.stderr) {{ error make {{ msg: "overlay error missing" }} }} +if not ("overlay use activate.nu" in $r1.stderr) {{ error make {{ msg: "hint missing for use-star" }} }} +let r2 = (^nu --commands $"overlay use ($path | to nuon) as myenv; deactivate" | complete) +if ($r2.exit_code == 0) {{ error make {{ msg: "expected deactivate to fail for custom name" }} }} +if not ("not an active overlay" in $r2.stderr) {{ error make {{ msg: "overlay error missing" }} }} +if not ("overlay hide NAME" in $r2.stderr) {{ error make {{ msg: "hint missing for custom name" }} }} +""" + result = subprocess.run( + [nu, "--commands", script], capture_output=True, text=True, encoding="utf-8", timeout=60, check=False + ) + assert result.returncode == 0, result.stderr diff --git a/tests/unit/activation/test_powershell.py b/tests/unit/activation/test_powershell.py index 2a48956cf..a51f8960b 100644 --- a/tests/unit/activation/test_powershell.py +++ b/tests/unit/activation/test_powershell.py @@ -1,12 +1,78 @@ from __future__ import annotations +import shutil +import subprocess import sys from argparse import Namespace +from typing import TYPE_CHECKING import pytest from virtualenv.activation import PowerShellActivator +if TYPE_CHECKING: + from pathlib import Path + + +@pytest.mark.skipif(shutil.which("pwsh") is None, reason="requires PowerShell Core") +@pytest.mark.parametrize( + "delimiter", + [ + pytest.param(chr(0x27), id="apostrophe"), + pytest.param(chr(0x2018), id="left-single"), + pytest.param(chr(0x2019), id="right-single"), + pytest.param(chr(0x201A), id="low-9"), + pytest.param(chr(0x201B), id="high-reversed-9"), + ], +) +def test_powershell_quote_neutralizes_single_quote_delimiters(tmp_path: Path, delimiter: str) -> None: + marker = tmp_path / "PWNED" + payload = f"x{delimiter}+$(New-Item -ItemType File -Path '{marker}')+{delimiter}y" + literal = PowerShellActivator.quote(payload) + result = subprocess.run( + ["pwsh", "-NoProfile", "-NonInteractive", "-Command", f"Write-Output {literal}"], + capture_output=True, + text=True, + encoding="utf-8", + check=False, + ) + + assert (marker.exists(), result.stdout.strip()) == (False, payload), result.stderr + + +def test_powershell_pydoc_call_operator(tmp_path) -> None: + """Test that PowerShell pydoc function uses call operator to handle spaces in python path.""" + + # GIVEN: A mock interpreter + class MockInterpreter: + os = "nt" + tcl_lib = None + tk_lib = None + + class MockCreator: + def __init__(self, dest) -> None: + self.dest = dest + self.bin_dir = dest / "Scripts" + self.bin_dir.mkdir(parents=True) + self.interpreter = MockInterpreter() + self.pyenv_cfg = {} + self.env_name = "test-env" + + creator = MockCreator(tmp_path) + options = Namespace(prompt=None) + activator = PowerShellActivator(options) + + # WHEN: Generate activation scripts + activator.generate(creator) + + # THEN: pydoc function should use call operator & to handle paths with spaces + activate_content = (creator.bin_dir / "activate.ps1").read_text(encoding="utf-8-sig") + + # The pydoc function should use & call operator to handle paths with spaces + assert "& python -m pydoc" in activate_content, ( + f"pydoc function should use & call operator. Content:\n{activate_content}" + ) + @pytest.mark.parametrize( ("tcl_lib", "tk_lib", "present"), @@ -15,7 +81,7 @@ (None, None, False), ], ) -def test_powershell_tkinter_generation(tmp_path, tcl_lib, tk_lib, present): +def test_powershell_tkinter_generation(tmp_path, tcl_lib, tk_lib, present) -> None: # GIVEN class MockInterpreter: os = "nt" @@ -25,7 +91,7 @@ class MockInterpreter: interpreter.tk_lib = tk_lib class MockCreator: - def __init__(self, dest): + def __init__(self, dest) -> None: self.dest = dest self.bin_dir = dest / "bin" self.bin_dir.mkdir() @@ -42,6 +108,13 @@ def __init__(self, dest): content = (creator.bin_dir / "activate.ps1").read_text(encoding="utf-8-sig") # THEN + # PKG_CONFIG_PATH is always set + assert "New-Variable -Scope global -Name _OLD_PKG_CONFIG_PATH" in content + assert '$env:PKG_CONFIG_PATH = [IO.Path]::Combine($env:VIRTUAL_ENV, "lib", "pkgconfig")' in content + assert "if (Test-Path variable:_OLD_PKG_CONFIG_PATH)" in content + assert "$env:PKG_CONFIG_PATH = $variable:_OLD_PKG_CONFIG_PATH" in content + assert 'Remove-Variable "_OLD_PKG_CONFIG_PATH" -Scope global' in content + if present: assert "if ('C:\\tcl' -ne \"\")" in content assert "$env:TCL_LIBRARY = 'C:\\tcl'" in content @@ -55,15 +128,15 @@ def __init__(self, dest): @pytest.mark.slow -def test_powershell(activation_tester_class, activation_tester, monkeypatch): +def test_powershell(activation_tester_class, activation_tester, monkeypatch) -> None: monkeypatch.setenv("TERM", "xterm") class PowerShell(activation_tester_class): def __init__(self, session) -> None: - cmd = "powershell.exe" if sys.platform == "win32" else "pwsh" + cmd = "pwsh" if shutil.which("pwsh") else "powershell.exe" if sys.platform == "win32" else "pwsh" super().__init__(PowerShellActivator, session, cmd, "activate.ps1", "ps1") self._version_cmd = [cmd, "-c", "$PSVersionTable"] - self._invoke_script = [cmd, "-ExecutionPolicy", "ByPass", "-File"] + self._invoke_script = [cmd, "-NonInteractive", "-NoProfile", "-ExecutionPolicy", "ByPass", "-File"] self.activate_cmd = "." self.script_encoding = "utf-8-sig" @@ -73,15 +146,15 @@ def _get_test_lines(self, activate_script): def invoke_script(self): return [self.cmd, "-File"] - def print_prompt(self): + def print_state(self, names: tuple[str, ...]) -> str: + lookups = (f'if ($env:{var} -eq $null) {{ "None" }} else {{ $env:{var} }}' for var in names) + return "; ".join([self.print_python_exe(), *lookups]) + + def print_prompt(self) -> str: return "prompt" def quote(self, s): - """ - Tester will pass strings to native commands on Windows so extra - parsing rules are used. Check `PowerShellActivator.quote` for more - details. - """ + """Tester will pass strings to native commands on Windows so extra parsing rules are used. Check `PowerShellActivator.quote` for more details.""" text = PowerShellActivator.quote(s) return text.replace('"', '""') if sys.platform == "win32" else text diff --git a/tests/unit/activation/test_python_activator.py b/tests/unit/activation/test_python_activator.py index 24a3561c5..b79d297cc 100644 --- a/tests/unit/activation/test_python_activator.py +++ b/tests/unit/activation/test_python_activator.py @@ -2,6 +2,7 @@ import os import sys +from argparse import Namespace from ast import literal_eval from textwrap import dedent @@ -9,7 +10,40 @@ from virtualenv.info import IS_WIN -def test_python(raise_on_non_source_class, activation_tester): +def test_python_activator_generates_pkg_config_path(tmp_path) -> None: + """Test that activate_this.py sets PKG_CONFIG_PATH.""" + + class MockInterpreter: + tcl_lib = None + tk_lib = None + + class MockCreator: + def __init__(self, dest) -> None: + self.dest = dest + self.bin_dir = dest / ("Scripts" if IS_WIN else "bin") + self.bin_dir.mkdir(parents=True) + self.libs = [dest / "Lib" / "site-packages"] + self.env_name = "test-env" + self.interpreter = MockInterpreter() + self.pyenv_cfg = {} + + creator = MockCreator(tmp_path) + options = Namespace(prompt=None) + activator = PythonActivator(options) + + # Generate the activation script + activator.generate(creator) + + # Read the generated script + content = (creator.bin_dir / "activate_this.py").read_text(encoding="utf-8") + + # Verify PKG_CONFIG_PATH is set + assert "PKG_CONFIG_PATH" in content + assert "pkg_config_path" in content + assert 'os.path.join(base, "lib", "pkgconfig")' in content + + +def test_python(raise_on_non_source_class, activation_tester) -> None: class Python(raise_on_non_source_class): def __init__(self, session) -> None: super().__init__( @@ -59,7 +93,7 @@ def print_r(value): """ return dedent(raw).splitlines() - def assert_output(self, out, raw, tmp_path): # noqa: ARG002 + def assert_output(self, out, raw, tmp_path) -> None: # ruff:ignore[unused-method-argument] out = [literal_eval(i) for i in out] assert out[0] is None # start with VIRTUAL_ENV None assert out[1] is None # likewise for VIRTUAL_ENV_PROMPT diff --git a/tests/unit/activation/test_xonsh.py b/tests/unit/activation/test_xonsh.py new file mode 100644 index 000000000..0612fd1a5 --- /dev/null +++ b/tests/unit/activation/test_xonsh.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +from argparse import Namespace +from shutil import which +from unittest.mock import create_autospec + +import pytest +from python_discovery import PythonInfo + +from virtualenv.activation import XonshActivator +from virtualenv.create.creator import Creator + + +@pytest.mark.parametrize( + ("tcl_lib", "tk_lib", "present"), + [ + ("/path/to/tcl", "/path/to/tk", True), + (None, None, False), + ], +) +def test_xonsh_tkinter_generation(tmp_path, tcl_lib, tk_lib, present) -> None: + # GIVEN + interpreter = create_autospec(PythonInfo, instance=True) + interpreter.tcl_lib = tcl_lib + interpreter.tk_lib = tk_lib + + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + + creator = create_autospec(Creator, instance=True) + creator.dest = tmp_path + creator.bin_dir = bin_dir + creator.interpreter = interpreter + creator.pyenv_cfg = {} + creator.env_name = "my-env" + + activator = XonshActivator(Namespace(prompt=None)) + + # WHEN + activator.generate(creator) + content = (bin_dir / "activate.xsh").read_text(encoding="utf-8") + + # THEN + # `managed_vars` always lists TCL_LIBRARY/TK_LIBRARY so deactivate can clean up + # whatever activation introduced — regardless of whether the interpreter reported + # tcl_lib/tk_lib. + assert '"TCL_LIBRARY"' in content + assert '"TK_LIBRARY"' in content + + if present: + # Paths are embedded in __init__ as Python string literals and used + # from self.embedded_* attributes during activate(). + assert "self.embedded_tcl_library = '/path/to/tcl'" in content + assert "self.embedded_tk_library = '/path/to/tk'" in content + else: + # Empty strings are falsy, so the override loop is skipped at runtime. + assert "self.embedded_tcl_library = ''" in content + assert "self.embedded_tk_library = ''" in content + + +def test_xonsh_quote() -> None: + # XonshActivator.quote uses repr() so the template substitution produces valid + # Python literals — this is what the rest of the tests rely on. + assert XonshActivator.quote("hello") == "'hello'" + assert XonshActivator.quote("it's") == '"it\'s"' + assert XonshActivator.quote("") == "''" + + +@pytest.mark.skipif(which("xonsh") is None, reason="xonsh is not installed") +def test_xonsh(activation_tester_class, activation_tester) -> None: + class Xonsh(activation_tester_class): + def __init__(self, session) -> None: + super().__init__(XonshActivator, session, "xonsh", "activate.xsh", "xsh") + self._invoke_script.append("--no-rc") + + def env(self, tmp_path): + env = super().env(tmp_path) + # Keep the subprocess hermetic: skip auto-loaded xontribs that a dev may + # have installed locally, and surface tracebacks so test failures are + # actionable. + env["XONTRIBS_AUTOLOAD_DISABLED"] = "1" + env["XONSH_SHOW_TRACEBACK"] = "1" + return env + + def print_prompt(self): + return 'echo @("(" + $VIRTUAL_ENV_PROMPT + ") ")' + + def assert_pkg_config_path(self, before, activated, deactivated, raw) -> None: + # activate.xsh does not manage PKG_CONFIG_PATH + assert before == activated == deactivated, raw + + def activate_call(self, script): + return f"source {self.quote(str(script))}" + + activation_tester(Xonsh) diff --git a/tests/unit/config/cli/test_help_formatter.py b/tests/unit/config/cli/test_help_formatter.py new file mode 100644 index 000000000..49e0f2424 --- /dev/null +++ b/tests/unit/config/cli/test_help_formatter.py @@ -0,0 +1,27 @@ +"""Tests for HelpFormatter terminal width behavior.""" + +from __future__ import annotations + +import os +from unittest.mock import patch + +import pytest + +from virtualenv.config.cli.parser import HelpFormatter + + +@pytest.mark.parametrize("width", [80, 120, 200]) +def test_help_formatter_uses_terminal_width(width: int) -> None: + """HelpFormatter should use shutil.get_terminal_size() for width.""" + with patch("virtualenv.config.cli.parser.shutil.get_terminal_size") as mock_size: + mock_size.return_value = os.terminal_size((width, 24)) + formatter = HelpFormatter("test_prog") + assert formatter._width == width # ruff:ignore[private-member-access] + + +def test_help_formatter_not_hardcoded_240() -> None: + """HelpFormatter width should not be hardcoded to 240.""" + with patch("virtualenv.config.cli.parser.shutil.get_terminal_size") as mock_size: + mock_size.return_value = os.terminal_size((80, 24)) + formatter = HelpFormatter("test_prog") + assert formatter._width != 240 # ruff:ignore[private-member-access] diff --git a/tests/unit/config/cli/test_parser.py b/tests/unit/config/cli/test_parser.py index 1dc7e055a..789c5b02e 100644 --- a/tests/unit/config/cli/test_parser.py +++ b/tests/unit/config/cli/test_parser.py @@ -24,13 +24,15 @@ def _build(): def _run(*args): return parser.parse_args(args=args) - yield parser, _run - parser.enable_help() + try: + yield parser, _run + finally: + parser.enable_help() return _build -def test_flag(gen_parser_no_conf_env): +def test_flag(gen_parser_no_conf_env) -> None: with gen_parser_no_conf_env() as (parser, run): parser.add_argument("--clear", dest="clear", action="store_true", help="it", default=False) result = run() @@ -39,14 +41,14 @@ def test_flag(gen_parser_no_conf_env): assert result.clear is True -def test_reset_app_data_does_not_conflict_clear(): +def test_reset_app_data_does_not_conflict_clear() -> None: options = VirtualEnvOptions() session_via_cli(["--clear", "venv"], options=options) assert options.clear is True assert options.reset_app_data is False -def test_builtin_discovery_class_preferred(mocker): +def test_builtin_discovery_class_preferred(mocker) -> None: mocker.patch( "virtualenv.run.plugin.discovery._get_default_discovery", return_value=["pluginA", "pluginX", "builtin", "Aplugin", "Xplugin"], diff --git a/tests/unit/config/test___main__.py b/tests/unit/config/test___main__.py index 0bd2569d6..8d2f21f4f 100644 --- a/tests/unit/config/test___main__.py +++ b/tests/unit/config/test___main__.py @@ -2,8 +2,8 @@ import re import sys -from subprocess import PIPE, Popen, check_output -from typing import TYPE_CHECKING +from subprocess import check_output +from typing import TYPE_CHECKING, NoReturn import pytest @@ -14,26 +14,14 @@ from pathlib import Path -def test_main(): - process = Popen( - [sys.executable, "-m", "virtualenv", "--help"], - universal_newlines=True, - stdout=PIPE, - encoding="utf-8", - ) - out, _ = process.communicate() - assert not process.returncode - assert out - - @pytest.fixture def raise_on_session_done(mocker): - def _func(exception): - from virtualenv.run import session_via_cli # noqa: PLC0415 + def _func(exception) -> None: + from virtualenv.run import session_via_cli # ruff:ignore[import-outside-top-level] prev_session = session_via_cli - def _session_via_cli(args, options=None, setup_logging=True, env=None): + def _session_via_cli(args, options=None, setup_logging=True, env=None) -> NoReturn: prev_session(args, options, setup_logging, env) raise exception @@ -42,7 +30,7 @@ def _session_via_cli(args, options=None, setup_logging=True, env=None): return _func -def test_fail_no_traceback(raise_on_session_done, tmp_path, capsys): +def test_fail_no_traceback(raise_on_session_done, tmp_path, capsys) -> None: raise_on_session_done(ProcessCallFailedError(code=2, out="out\n", err="err\n", cmd=["something"])) with pytest.raises(SystemExit) as context: run_with_catch([str(tmp_path)]) @@ -52,7 +40,7 @@ def test_fail_no_traceback(raise_on_session_done, tmp_path, capsys): assert err == "err\n" -def test_discovery_fails_no_discovery_plugin(mocker, tmp_path, capsys): +def test_discovery_fails_no_discovery_plugin(mocker, tmp_path, capsys) -> None: mocker.patch("virtualenv.run.plugin.discovery.Discovery.entry_points_for", return_value={}) with pytest.raises(SystemExit) as context: run_with_catch([str(tmp_path)]) @@ -62,7 +50,7 @@ def test_discovery_fails_no_discovery_plugin(mocker, tmp_path, capsys): assert not err -def test_fail_with_traceback(raise_on_session_done, tmp_path, capsys): +def test_fail_with_traceback(raise_on_session_done, tmp_path, capsys) -> None: raise_on_session_done(TypeError("something bad")) with pytest.raises(TypeError, match="something bad"): @@ -88,14 +76,14 @@ def test_session_report_full(tmp_path: Path, capsys: pytest.CaptureFixture[str]) _match_regexes(lines, regexes) -def _match_regexes(lines, regexes): +def _match_regexes(lines, regexes) -> None: for line, regex in zip(lines, regexes): comp_regex = re.compile(rf"^{regex}$") assert comp_regex.match(line), line @pytest.mark.usefixtures("session_app_data") -def test_session_report_minimal(tmp_path, capsys): +def test_session_report_minimal(tmp_path, capsys) -> None: run_with_catch([str(tmp_path), "--activators", "", "--without-pip"]) out, err = capsys.readouterr() assert not err @@ -108,7 +96,7 @@ def test_session_report_minimal(tmp_path, capsys): @pytest.mark.usefixtures("session_app_data") -def test_session_report_subprocess(tmp_path): +def test_session_report_subprocess(tmp_path) -> None: # when called via a subprocess the logging framework should flush and POSIX line normalization happen out = check_output( [sys.executable, "-m", "virtualenv", str(tmp_path), "--activators", "powershell", "--without-pip"], diff --git a/tests/unit/config/test_env_var.py b/tests/unit/config/test_env_var.py index 5f364978d..37c3a5f55 100644 --- a/tests/unit/config/test_env_var.py +++ b/tests/unit/config/test_env_var.py @@ -4,30 +4,31 @@ from pathlib import Path import pytest +from python_discovery import PythonInfo from virtualenv.config.cli.parser import VirtualEnvOptions +from virtualenv.config.convert import ListType from virtualenv.config.ini import IniConfig from virtualenv.create.via_global_ref.builtin.cpython.common import is_macos_brew -from virtualenv.discovery.py_info import PythonInfo from virtualenv.run import session_via_cli @pytest.fixture -def _empty_conf(tmp_path, monkeypatch): +def _empty_conf(tmp_path, monkeypatch) -> None: conf = tmp_path / "conf.ini" monkeypatch.setenv(IniConfig.VIRTUALENV_CONFIG_FILE_ENV_VAR, str(conf)) conf.write_text("[virtualenv]", encoding="utf-8") @pytest.mark.usefixtures("_empty_conf") -def test_value_ok(monkeypatch): +def test_value_ok(monkeypatch) -> None: monkeypatch.setenv("VIRTUALENV_VERBOSE", "5") result = session_via_cli(["venv"]) assert result.verbosity == 5 @pytest.mark.usefixtures("_empty_conf") -def test_value_bad(monkeypatch, caplog): +def test_value_bad(monkeypatch, caplog) -> None: monkeypatch.setenv("VIRTUALENV_VERBOSE", "a") result = session_via_cli(["venv"]) assert result.verbosity == 2 @@ -36,35 +37,33 @@ def test_value_bad(monkeypatch, caplog): assert "invalid literal" in caplog.messages[0] -def test_python_via_env_var(monkeypatch): - options = VirtualEnvOptions() - monkeypatch.setenv("VIRTUALENV_PYTHON", "python3") - session_via_cli(["venv"], options=options) - assert options.python == ["python3"] - - -def test_python_multi_value_via_env_var(monkeypatch): - options = VirtualEnvOptions() - monkeypatch.setenv("VIRTUALENV_PYTHON", "python3,python2") - session_via_cli(["venv"], options=options) - assert options.python == ["python3", "python2"] - - -def test_python_multi_value_newline_via_env_var(monkeypatch): - options = VirtualEnvOptions() - monkeypatch.setenv("VIRTUALENV_PYTHON", "python3\npython2") - session_via_cli(["venv"], options=options) - assert options.python == ["python3", "python2"] - - -def test_python_multi_value_prefer_newline_via_env_var(monkeypatch): +def test_python_multi_value_prefer_newline_via_env_var(monkeypatch) -> None: options = VirtualEnvOptions() monkeypatch.setenv("VIRTUALENV_PYTHON", "python3\npython2,python27") session_via_cli(["venv"], options=options) assert options.python == ["python3", "python2,python27"] -def test_extra_search_dir_via_env_var(tmp_path, monkeypatch): +@pytest.mark.parametrize( + ("value", "expected"), + [ + pytest.param("a,b,c", ["a", "b", "c"], id="comma"), + pytest.param("a\nb\nc", ["a", "b", "c"], id="newline"), + pytest.param("a\nb,c", ["a", "b,c"], id="newline_wins_over_comma"), + pytest.param(" a , b ", ["a", "b"], id="strips_whitespace"), + pytest.param("a\n\nb", ["a", "b"], id="drops_blank_lines"), + pytest.param("a", ["a"], id="single"), + pytest.param("", [], id="empty"), + pytest.param(b"a,b", ["a", "b"], id="bytes_comma"), + pytest.param(b"a\nb", ["a", "b"], id="bytes_newline"), + pytest.param(["a", "b"], ["a", "b"], id="list_passthrough"), + ], +) +def test_split_values(value, expected) -> None: + assert ListType(list, str).split_values(value) == expected + + +def test_extra_search_dir_via_env_var(tmp_path, monkeypatch) -> None: monkeypatch.chdir(tmp_path) value = f"a{os.linesep}0{os.linesep}b{os.pathsep}c" monkeypatch.setenv("VIRTUALENV_EXTRA_SEARCH_DIR", str(value)) @@ -77,10 +76,10 @@ def test_extra_search_dir_via_env_var(tmp_path, monkeypatch): @pytest.mark.usefixtures("_empty_conf") @pytest.mark.skipif(is_macos_brew(PythonInfo.current_system()), reason="no copy on brew") -def test_value_alias(monkeypatch, mocker): - from virtualenv.config.cli.parser import VirtualEnvConfigParser # noqa: PLC0415 +def test_value_alias(monkeypatch, mocker) -> None: + from virtualenv.config.cli.parser import VirtualEnvConfigParser # ruff:ignore[import-outside-top-level] - prev = VirtualEnvConfigParser._fix_default # noqa: SLF001 + prev = VirtualEnvConfigParser._fix_default # ruff:ignore[private-member-access] def func(self, action): if action.dest == "symlinks": diff --git a/tests/unit/config/test_ini.py b/tests/unit/config/test_ini.py index a1621fa30..44f844ef4 100644 --- a/tests/unit/config/test_ini.py +++ b/tests/unit/config/test_ini.py @@ -1,13 +1,22 @@ from __future__ import annotations +import codecs +import logging import sys -from textwrap import dedent +from pathlib import Path +from typing import TYPE_CHECKING, Final, cast import pytest +from virtualenv.config.ini import IniConfig from virtualenv.info import IS_PYPY, IS_WIN, fs_supports_symlink from virtualenv.run import session_via_cli +if TYPE_CHECKING: + from pytest_mock import MockerFixture + + from virtualenv.create.via_global_ref.api import ViaGlobalRefApi + @pytest.mark.skipif(not fs_supports_symlink(), reason="symlink is not supported") @pytest.mark.xfail( @@ -15,20 +24,74 @@ IS_PYPY and IS_WIN and sys.version_info[0:2] >= (3, 9), reason="symlink is not supported", ) -def test_ini_can_be_overwritten_by_flag(tmp_path, monkeypatch): - custom_ini = tmp_path / "conf.ini" - custom_ini.write_text( - dedent( - """ - [virtualenv] - copies = True - """, - ), - encoding="utf-8", - ) +def test_ini_can_be_overwritten_by_flag(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + custom_ini: Final[Path] = tmp_path / "conf.ini" + custom_ini.write_text("[virtualenv]\ncopies = True\n", encoding="utf-8") monkeypatch.setenv("VIRTUALENV_CONFIG_FILE", str(custom_ini)) - result = session_via_cli(["venv", "--symlinks"]) + assert cast("ViaGlobalRefApi", session_via_cli(["venv", "--symlinks"]).creator).symlinks is True + + +def test_ini_that_fails_to_parse_is_ignored(invalid_ini: Path) -> None: + config: Final[IniConfig] = IniConfig(env={"VIRTUALENV_CONFIG_FILE": str(invalid_ini)}) + assert (bool(config), config.has_virtualenv_section) == (False, False) + + +def test_ini_that_fails_to_parse_is_reported(invalid_ini: Path) -> None: + config: Final[IniConfig] = IniConfig(env={"VIRTUALENV_CONFIG_FILE": str(invalid_ini)}) + assert config.epilog == ( + f"\nconfig file {invalid_ini.resolve()} failed to parse (changed via env var VIRTUALENV_CONFIG_FILE)" + ) + + +def test_ini_that_fails_to_parse_is_logged(invalid_ini: Path, caplog: pytest.LogCaptureFixture) -> None: + IniConfig(env={"VIRTUALENV_CONFIG_FILE": str(invalid_ini)}) + assert [(record.levelno, record.message.split(" because ")[0]) for record in caplog.records] == [ + (logging.ERROR, f"failed to read config file {invalid_ini}") + ] + + +def test_ini_that_fails_to_parse_does_not_break_the_cli(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + # configparser reads clear = True before it hits the bad line, and the CLI must not apply that partial result + (invalid_ini := tmp_path / "conf.ini").write_bytes(b"[virtualenv]\nclear = True\ninvalid\n") + monkeypatch.setenv("VIRTUALENV_CONFIG_FILE", str(invalid_ini)) + + assert session_via_cli(["venv"]).creator.clear is False + + +@pytest.fixture( + params=[ + pytest.param(b"no section header\n", id="missing-header"), + pytest.param(b"[virtualenv]\nclear = True\ninvalid\n", id="partial-parse"), + pytest.param(b"[virtualenv]\nclear = True\nclear = False\n", id="duplicate-option"), + pytest.param(b"[virtualenv]\nclear = True\n[virtualenv]\n", id="duplicate-section"), + pytest.param(b"[virtualenv]\nclear = True\n\xff", id="invalid-utf8"), + ] +) +def invalid_ini(tmp_path: Path, request: pytest.FixtureRequest) -> Path: + config_file: Final[Path] = tmp_path / "conf.ini" + config_file.write_bytes(request.param) + return config_file + + +@pytest.mark.parametrize( + "method", + [pytest.param("exists", id="stat"), pytest.param("resolve", id="resolve"), pytest.param("open", id="read")], +) +def test_ini_filesystem_error_is_ignored(tmp_path: Path, mocker: MockerFixture, method: str) -> None: + config_file: Final[Path] = tmp_path / "conf.ini" + config_file.write_text("[virtualenv]\nclear = True\n", encoding="utf-8") + mocker.patch.object(Path, method, autospec=True, side_effect=PermissionError("access denied")) + + config: Final[IniConfig] = IniConfig(env={"VIRTUALENV_CONFIG_FILE": str(config_file)}) + + assert (bool(config), config.has_config_file, config.has_virtualenv_section) == (False, None, False) + + +@pytest.mark.parametrize("prefix", [pytest.param(b"", id="utf8"), pytest.param(codecs.BOM_UTF8, id="utf8-bom")]) +def test_ini_utf8_is_read(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, prefix: bytes) -> None: + config_file: Final[Path] = tmp_path / "conf.ini" + config_file.write_bytes(prefix + b"[virtualenv]\nclear = True\n") + monkeypatch.setenv("VIRTUALENV_CONFIG_FILE", str(config_file)) - symlinks = result.creator.symlinks - assert symlinks is True + assert session_via_cli(["venv"]).creator.clear is True diff --git a/tests/unit/create/conftest.py b/tests/unit/create/conftest.py index 58d390c5c..a1339f22f 100644 --- a/tests/unit/create/conftest.py +++ b/tests/unit/create/conftest.py @@ -1,10 +1,10 @@ -""" -It's possible to use multiple types of host pythons to create virtual environments and all should work: +"""It's possible to use multiple types of host pythons to create virtual environments and all should work: - host installation - invoking from a venv (if Python 3.3+) - invoking from an old style virtualenv (<17.0.0) - invoking from our own venv + """ from __future__ import annotations @@ -13,13 +13,12 @@ from subprocess import Popen import pytest - -from virtualenv.discovery.py_info import PythonInfo +from python_discovery import PythonInfo CURRENT = PythonInfo.current_system() -def root(tmp_path_factory, session_app_data): # noqa: ARG001 +def root(tmp_path_factory, session_app_data): # ruff:ignore[unused-function-argument] return CURRENT.system_executable diff --git a/tests/unit/create/console_app/demo/__init__.py b/tests/unit/create/console_app/demo/__init__.py index d7f2575eb..f55ba6cda 100644 --- a/tests/unit/create/console_app/demo/__init__.py +++ b/tests/unit/create/console_app/demo/__init__.py @@ -1,8 +1,8 @@ from __future__ import annotations -def run(): - print("magic") # noqa: T201 +def run() -> None: + print("magic") # ruff:ignore[print] if __name__ == "__main__": diff --git a/tests/unit/create/console_app/demo/__main__.py b/tests/unit/create/console_app/demo/__main__.py index d7f2575eb..f55ba6cda 100644 --- a/tests/unit/create/console_app/demo/__main__.py +++ b/tests/unit/create/console_app/demo/__main__.py @@ -1,8 +1,8 @@ from __future__ import annotations -def run(): - print("magic") # noqa: T201 +def run() -> None: + print("magic") # ruff:ignore[print] if __name__ == "__main__": diff --git a/tests/unit/create/test_creator.py b/tests/unit/create/test_creator.py index c72e830bc..fb4d6d843 100644 --- a/tests/unit/create/test_creator.py +++ b/tests/unit/create/test_creator.py @@ -19,23 +19,27 @@ from stat import S_IREAD, S_IRGRP, S_IROTH from textwrap import dedent from threading import Thread +from typing import TYPE_CHECKING, Final import pytest +from python_discovery import PythonInfo from virtualenv.__main__ import run, run_with_catch from virtualenv.create.creator import DEBUG_SCRIPT, Creator, get_env_debug_info from virtualenv.create.pyenv_cfg import PyEnvCfg from virtualenv.create.via_global_ref import api -from virtualenv.create.via_global_ref.builtin.cpython.common import is_macos_brew -from virtualenv.create.via_global_ref.builtin.cpython.cpython3 import CPython3Posix -from virtualenv.discovery.py_info import PythonInfo -from virtualenv.info import IS_PYPY, IS_WIN, fs_is_case_sensitive +from virtualenv.create.via_global_ref.builtin.cpython.common import is_mac_os_framework, is_macos_brew +from virtualenv.info import IS_PYPY, IS_WIN, fs_is_case_sensitive, fs_supports_symlink from virtualenv.run import cli_run, session_via_cli +from virtualenv.run.plugin.creators import CreatorSelector + +if TYPE_CHECKING: + from virtualenv.run.session import Session CURRENT = PythonInfo.current_system() -def test_os_path_sep_not_allowed(tmp_path, capsys): +def test_os_path_sep_not_allowed(tmp_path, capsys) -> None: target = str(tmp_path / f"a{os.pathsep}b") err = _non_success_exit_code(capsys, target) msg = ( @@ -54,7 +58,7 @@ def _non_success_exit_code(capsys, target): return err -def test_destination_exists_file(tmp_path, capsys): +def test_destination_exists_file(tmp_path, capsys) -> None: target = tmp_path / "out" target.write_text("", encoding="utf-8") err = _non_success_exit_code(capsys, str(target)) @@ -63,7 +67,7 @@ def test_destination_exists_file(tmp_path, capsys): @pytest.mark.skipif(sys.platform == "win32", reason="Windows only applies R/O to files") -def test_destination_not_write_able(tmp_path, capsys): +def test_destination_not_write_able(tmp_path, capsys) -> None: if hasattr(os, "geteuid") and os.geteuid() == 0: pytest.skip("no way to check permission restriction when running under root") @@ -79,7 +83,7 @@ def test_destination_not_write_able(tmp_path, capsys): def cleanup_sys_path(paths): - from virtualenv.create.creator import HERE # noqa: PLC0415 + from virtualenv.create.creator import HERE # ruff:ignore[import-outside-top-level] paths = [p.resolve() for p in (Path(os.path.abspath(i)) for i in paths) if p.exists()] to_remove = [Path(HERE)] @@ -93,9 +97,9 @@ def system(session_app_data): return get_env_debug_info(Path(CURRENT.system_executable), DEBUG_SCRIPT, session_app_data, os.environ) -CURRENT_CREATORS = [i for i in CURRENT.creators().key_to_class if i != "builtin"] +CURRENT_CREATORS = [i for i in CreatorSelector.for_interpreter(CURRENT).key_to_class if i != "builtin"] CREATE_METHODS = [] -for k, v in CURRENT.creators().key_to_meta.items(): +for k, v in CreatorSelector.for_interpreter(CURRENT).key_to_meta.items(): if k in CURRENT_CREATORS: if v.can_copy: if k == "venv" and CURRENT.implementation == "PyPy" and CURRENT.pypy_version_info >= [7, 3, 13]: @@ -105,18 +109,19 @@ def system(session_app_data): CREATE_METHODS.append((k, "symlinks")) +@pytest.mark.graalpy @pytest.mark.parametrize( ("creator", "isolated"), [pytest.param(*i, id=f"{'-'.join(i[0])}-{i[1]}") for i in product(CREATE_METHODS, ["isolated", "global"])], ) -def test_create_no_seed( # noqa: C901, PLR0912, PLR0913, PLR0915 +def test_create_no_seed( # ruff:ignore[complex-structure, too-many-branches, too-many-arguments, too-many-statements] python, creator, isolated, system, coverage_env, special_name_dir, -): +) -> None: dest = special_name_dir creator_key, method = creator cmd = [ @@ -210,22 +215,11 @@ def list_to_str(iterable): assert python_w.exists() assert python_w.read_bytes() != creator.exe.read_bytes() - if CPython3Posix.pyvenv_launch_patch_active(PythonInfo.from_exe(python)) and creator_key != "venv": - result = subprocess.check_output( - [str(creator.exe), "-c", 'import os; print(os.environ.get("__PYVENV_LAUNCHER__"))'], - text=True, - ).strip() - assert result == "None" - git_ignore = (dest / ".gitignore").read_text(encoding="utf-8") - if creator_key == "venv" and sys.version_info >= (3, 13): - comment = "# Created by venv; see https://docs.python.org/3/library/venv.html" - else: - comment = "# created by virtualenv automatically" - assert git_ignore.splitlines() == [comment, "*"] + assert git_ignore.splitlines() == ["# created by virtualenv automatically", "*"] -def test_create_cachedir_tag(tmp_path): +def test_create_cachedir_tag(tmp_path) -> None: cachedir_tag_file = tmp_path / "CACHEDIR.TAG" cli_run([str(tmp_path), "--without-pip", "--activators", ""]) @@ -245,27 +239,20 @@ def test_create_cachedir_tag_exists(tmp_path: Path) -> None: assert cachedir_tag_file.read_text(encoding="utf-8") == "magic" -def test_create_cachedir_tag_exists_override(tmp_path: Path) -> None: - cachedir_tag_file = tmp_path / "CACHEDIR.TAG" - cachedir_tag_file.write_text("magic", encoding="utf-8") - cli_run([str(tmp_path), "--without-pip", "--activators", ""]) - assert cachedir_tag_file.read_text(encoding="utf-8") == "magic" - - -def test_create_vcs_ignore_exists(tmp_path): +def test_create_vcs_ignore_exists(tmp_path) -> None: git_ignore = tmp_path / ".gitignore" git_ignore.write_text("magic", encoding="utf-8") cli_run([str(tmp_path), "--without-pip", "--activators", ""]) assert git_ignore.read_text(encoding="utf-8") == "magic" -def test_create_vcs_ignore_override(tmp_path): +def test_create_vcs_ignore_override(tmp_path) -> None: git_ignore = tmp_path / ".gitignore" cli_run([str(tmp_path), "--without-pip", "--no-vcs-ignore", "--activators", ""]) assert not git_ignore.exists() -def test_create_vcs_ignore_exists_override(tmp_path): +def test_create_vcs_ignore_exists_override(tmp_path) -> None: git_ignore = tmp_path / ".gitignore" git_ignore.write_text("magic", encoding="utf-8") cli_run([str(tmp_path), "--without-pip", "--no-vcs-ignore", "--activators", ""]) @@ -273,7 +260,7 @@ def test_create_vcs_ignore_exists_override(tmp_path): @pytest.mark.skipif(not CURRENT.has_venv, reason="requires interpreter with venv") -def test_venv_fails_not_inline(tmp_path, capsys, mocker): +def test_venv_fails_not_inline(tmp_path, capsys, mocker) -> None: if hasattr(os, "geteuid") and os.geteuid() == 0: pytest.skip("no way to check permission restriction when running under root") @@ -302,7 +289,7 @@ def _session_via_cli(args, options=None, setup_logging=True, env=None): @pytest.mark.parametrize("creator", CURRENT_CREATORS) @pytest.mark.parametrize("clear", [True, False], ids=["clear", "no_clear"]) -def test_create_clear_resets(tmp_path, creator, clear, caplog): +def test_create_clear_resets(tmp_path, creator, clear, caplog) -> None: caplog.set_level(logging.DEBUG) if creator == "venv" and clear is False: pytest.skip("venv without clear might fail") @@ -318,28 +305,70 @@ def test_create_clear_resets(tmp_path, creator, clear, caplog): @pytest.mark.parametrize("creator", CURRENT_CREATORS) -@pytest.mark.parametrize("prompt", [None, "magic"]) -def test_prompt_set(tmp_path, creator, prompt): - cmd = [str(tmp_path), "--seeder", "app-data", "--without-pip", "--creator", creator] - if prompt is not None: - cmd.extend(["--prompt", "magic"]) - +@pytest.mark.parametrize("prompt", ["magic", "."]) +def test_prompt_set(tmp_path: Path, creator: str, prompt: str, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.chdir(tmp_path) + cmd = [str(tmp_path / "env"), "--seeder", "app-data", "--without-pip", "--creator", creator, "--prompt", prompt] result = cli_run(cmd) - actual_prompt = tmp_path.name if prompt is None else prompt cfg = PyEnvCfg.from_file(result.creator.pyenv_cfg.path) - if prompt is None: - assert "prompt" not in cfg - elif creator != "venv": + if creator != "venv": + expected = tmp_path.name if prompt == "." else prompt assert "prompt" in cfg, list(cfg.content.keys()) - assert cfg["prompt"] == actual_prompt + assert cfg["prompt"] == expected -@pytest.mark.parametrize("creator", CURRENT_CREATORS) -def test_home_path_is_exe_parent(tmp_path, creator): - cmd = [str(tmp_path), "--seeder", "app-data", "--without-pip", "--creator", creator] +# the tests below only read the environment, so each creator builds it once per module +@pytest.fixture(scope="module") +def default_env(request: pytest.FixtureRequest, tmp_path_factory: pytest.TempPathFactory) -> Session: + dest = tmp_path_factory.mktemp(f"default-{request.param}") + return cli_run([ + str(dest), + "--seeder", + "app-data", + "--without-pip", + "--creator", + request.param, + "--no-periodic-update", + ]) - result = cli_run(cmd) - cfg = PyEnvCfg.from_file(result.creator.pyenv_cfg.path) + +@pytest.mark.parametrize("default_env", CURRENT_CREATORS, indirect=True) +def test_prompt_not_set_by_default(default_env: Session) -> None: + assert "prompt" not in PyEnvCfg.from_file(default_env.creator.pyenv_cfg.path) + + +@pytest.mark.parametrize("default_env", CURRENT_CREATORS, indirect=True) +def test_version_keys_in_pyenv_cfg(default_env: Session) -> None: + cfg = PyEnvCfg.from_file(default_env.creator.pyenv_cfg.path) + version_info = default_env.creator.interpreter.version_info + assert cfg["python-version"] == f"{version_info.major}.{version_info.minor}" + assert "version" in cfg + parts = cfg["version"].split(".") + assert len(parts) == 3 + assert all(p.isdigit() for p in parts) + + +@pytest.mark.parametrize("default_env", [c for c in CURRENT_CREATORS if c != "venv"], indirect=True) +def test_executable_and_command_keys(default_env: Session) -> None: + cfg = PyEnvCfg.from_file(default_env.creator.pyenv_cfg.path) + assert "executable" in cfg + assert Path(cfg["executable"]).exists() + assert "command" in cfg + assert "virtualenv" in cfg["command"] + + +@pytest.mark.parametrize("default_env", [c for c in CURRENT_CREATORS if c != "venv"], indirect=True) +def test_include_dir_created(default_env: Session) -> None: + if sys.platform == "win32": + include = default_env.creator.dest / "Include" + else: + include = default_env.creator.dest / "include" + assert include.is_dir() + + +@pytest.mark.parametrize("default_env", CURRENT_CREATORS, indirect=True) +def test_home_path_is_exe_parent(default_env: Session) -> None: + cfg = PyEnvCfg.from_file(default_env.creator.pyenv_cfg.path) # Cannot assume "home" path is a specific value as path resolution may change # between versions (symlinks, framework paths, etc) but we can check that a @@ -356,9 +385,59 @@ def test_home_path_is_exe_parent(tmp_path, creator): assert any(os.path.exists(os.path.join(cfg["home"], exe)) for exe in exes) +@pytest.mark.skipif(IS_WIN or not fs_supports_symlink(), reason="requires POSIX interpreter aliases") +@pytest.mark.parametrize( + "mode", + [ + pytest.param( + "--copies", + id="copy", + marks=pytest.mark.skipif( + is_macos_brew(CURRENT) or is_mac_os_framework(CURRENT), + reason="Homebrew and framework builds require symlinks", + ), + ), + pytest.param("--symlinks", id="symlink"), + ], +) +def test_recreate_environment_with_dangling_alias(tmp_path: Path, mode: str) -> None: + destination: Final[Path] = tmp_path / "venv" + cli_run([str(destination), "--creator", "builtin", "--no-seed", "--symlinks"]) + alias: Final[Path] = destination / "bin" / f"python{sys.version_info.major}" + alias.unlink() + outside: Final[Path] = tmp_path / "removed-python" + alias.symlink_to(outside) + + cli_run([str(destination), "--creator", "builtin", "--no-seed", mode]) + + assert (alias.is_symlink(), alias.is_file(), outside.exists()) == (mode == "--symlinks", True, False) + + +@pytest.mark.skipif(sys.platform == "win32", reason="POSIX only") +@pytest.mark.skipif( + bool(CURRENT.sysconfig_vars.get("PYTHONFRAMEWORK")), + reason="framework builds self-locate via dyld and keep the recorded path", +) @pytest.mark.usefixtures("temp_app_data") -def test_create_parallel(tmp_path): - def create(count): +def test_home_resolves_executable_only_symlink(tmp_path: Path) -> None: + """An executable-only symlink must not be recorded as home / base-executable (issue #3157).""" + system_exe = CURRENT.system_executable + assert system_exe is not None + link = tmp_path / "symdir" / "python3" + link.parent.mkdir() + link.symlink_to(system_exe) + + result = cli_run(["-p", str(link), str(tmp_path / "env"), "--seeder", "app-data", "--without-pip"]) + cfg = PyEnvCfg.from_file(result.creator.pyenv_cfg.path) + + assert Path(cfg["home"]) != link.parent + assert os.path.samefile(cfg["base-executable"], system_exe) + assert Path(cfg["home"]) == Path(cfg["base-executable"]).parent + + +@pytest.mark.usefixtures("temp_app_data") +def test_create_parallel(tmp_path) -> None: + def create(count) -> None: subprocess.check_call( [sys.executable, "-m", "virtualenv", "-vvv", str(tmp_path / f"venv{count}"), "--without-pip"], ) @@ -370,21 +449,21 @@ def create(count): thread.join() -def test_creator_input_passed_is_abs(tmp_path, monkeypatch): +def test_creator_input_passed_is_abs(tmp_path, monkeypatch) -> None: monkeypatch.chdir(tmp_path) result = Creator.validate_dest("venv") assert str(result) == str(tmp_path / "venv") @pytest.mark.skipif(os.altsep is None, reason="OS does not have an altsep") -def test_creator_replaces_altsep_in_dest(tmp_path): +def test_creator_replaces_altsep_in_dest(tmp_path) -> None: dest = str(tmp_path / "venv{}foobar") result = Creator.validate_dest(dest.format(os.altsep)) assert str(result) == dest.format(os.sep) @pytest.mark.usefixtures("current_fastest") -def test_create_long_path(tmp_path): +def test_create_long_path(tmp_path) -> None: if sys.platform == "darwin": max_shebang_length = 512 else: @@ -394,15 +473,17 @@ def test_create_long_path(tmp_path): folder = tmp_path / ("a" * (count // 2)) / ("b" * (count // 2)) / "c" folder.mkdir(parents=True) - cmd = [str(folder)] + cmd = [str(folder), "--without-pip"] result = cli_run(cmd) - subprocess.check_call([str(result.creator.script("pip")), "--version"]) + subprocess.check_call([str(result.creator.exe), "--version"]) @pytest.mark.slow -@pytest.mark.parametrize("creator", sorted(set(PythonInfo.current_system().creators().key_to_class) - {"builtin"})) +@pytest.mark.parametrize( + "creator", sorted(set(CreatorSelector.for_interpreter(PythonInfo.current_system()).key_to_class) - {"builtin"}) +) @pytest.mark.usefixtures("session_app_data") -def test_create_distutils_cfg(creator, tmp_path, monkeypatch): +def test_create_distutils_cfg(creator, tmp_path, monkeypatch) -> None: result = cli_run( [ str(tmp_path / "venv"), @@ -451,6 +532,23 @@ def test_create_distutils_cfg(creator, tmp_path, monkeypatch): assert package_folder.exists(), list_files(str(tmp_path)) +def test_install_patch_skipped_on_py310(tmp_path, mocker) -> None: + creator = mocker.MagicMock(purelib=tmp_path, interpreter=mocker.MagicMock(version_info=(3, 10, 0))) + creator.env_patch_text.return_value = "import _virtualenv" + api.ViaGlobalRefApi.install_patch(creator) + assert not (tmp_path / "_virtualenv.pth").exists() + assert not (tmp_path / "_virtualenv.py").exists() + creator.env_patch_text.assert_not_called() + + +def test_install_patch_written_on_py39(tmp_path, mocker) -> None: + creator = mocker.MagicMock(purelib=tmp_path, interpreter=mocker.MagicMock(version_info=(3, 9, 21))) + creator.env_patch_text.return_value = "import _virtualenv" + api.ViaGlobalRefApi.install_patch(creator) + assert (tmp_path / "_virtualenv.pth").read_text(encoding="utf-8") == "import _virtualenv" + assert (tmp_path / "_virtualenv.py").read_text(encoding="utf-8") == "import _virtualenv" + + def list_files(path): result = "" for root, _, files in os.walk(path): @@ -465,7 +563,7 @@ def list_files(path): @pytest.mark.skipif(is_macos_brew(CURRENT), reason="no copy on brew") @pytest.mark.skip(reason="https://github.com/pypa/setuptools/issues/4640") -def test_zip_importer_can_import_setuptools(tmp_path): +def test_zip_importer_can_import_setuptools(tmp_path) -> None: """We're patching the loaders so might fail on r/o loaders, such as zipimporter on CPython<3.8""" result = cli_run( [str(tmp_path / "venv"), "--activators", "", "--no-pip", "--no-wheel", "--copies", "--setuptools", "bundle"], @@ -498,7 +596,7 @@ def test_zip_importer_can_import_setuptools(tmp_path): reason="https://foss.heptapod.net/pypy/pypy/-/issues/3269", ) @pytest.mark.usefixtures("_no_coverage") -def test_no_preimport_threading(tmp_path): +def test_no_preimport_threading(tmp_path) -> None: session = cli_run([str(tmp_path)]) out = subprocess.check_output( [str(session.creator.exe), "-c", r"import sys; print('\n'.join(sorted(sys.modules)))"], @@ -510,8 +608,8 @@ def test_no_preimport_threading(tmp_path): # verify that .pth files in site-packages/ are always processed even if $PYTHONPATH points to it. -def test_pth_in_site_vs_python_path(tmp_path): - session = cli_run([str(tmp_path)]) +def test_pth_in_site_vs_python_path(tmp_path) -> None: + session = cli_run([str(tmp_path), "--without-pip"]) site_packages = session.creator.purelib # install test.pth that sets sys.testpth='ok' (session.creator.purelib / "test.pth").write_text('import sys; sys.testpth="ok"\n', encoding="utf-8") @@ -537,9 +635,9 @@ def test_pth_in_site_vs_python_path(tmp_path): assert out == "ok\n" -def test_getsitepackages_system_site(tmp_path): +def test_getsitepackages_system_site(tmp_path) -> None: # Test without --system-site-packages - session = cli_run([str(tmp_path)]) + session = cli_run([str(tmp_path), "--without-pip"]) system_site_packages = get_expected_system_site_packages(session) @@ -553,8 +651,15 @@ def test_getsitepackages_system_site(tmp_path): for system_site_package in system_site_packages: assert system_site_package not in site_packages + env_site_packages = [str(session.creator.purelib), str(session.creator.platlib)] + if not fs_is_case_sensitive(): + env_site_packages = [x.lower() for x in env_site_packages] + site_packages = [x.lower() for x in site_packages] + for env_site_package in env_site_packages: + assert env_site_package in site_packages + # Test with --system-site-packages - session = cli_run([str(tmp_path), "--system-site-packages"]) + session = cli_run([str(tmp_path), "--system-site-packages", "--without-pip"]) system_site_packages = [str(Path(i).resolve()) for i in get_expected_system_site_packages(session)] @@ -580,26 +685,7 @@ def get_expected_system_site_packages(session): return system_site_packages -def test_get_site_packages(tmp_path): - case_sensitive = fs_is_case_sensitive() - session = cli_run([str(tmp_path)]) - env_site_packages = [str(session.creator.purelib), str(session.creator.platlib)] - out = subprocess.check_output( - [str(session.creator.exe), "-c", r"import site; print(site.getsitepackages())"], - text=True, - encoding="utf-8", - ) - site_packages = ast.literal_eval(out) - - if not case_sensitive: - env_site_packages = [x.lower() for x in env_site_packages] - site_packages = [x.lower() for x in site_packages] - - for env_site_package in env_site_packages: - assert env_site_package in site_packages - - -def test_debug_bad_virtualenv(tmp_path): +def test_debug_bad_virtualenv(tmp_path) -> None: cmd = [str(tmp_path), "--without-pip"] result = cli_run(cmd) # if the site.py is removed/altered the debug should fail as no one is around to fix the paths @@ -615,8 +701,9 @@ def test_debug_bad_virtualenv(tmp_path): assert debug_info["exception"] +@pytest.mark.graalpy @pytest.mark.parametrize("python_path_on", [True, False], ids=["on", "off"]) -def test_python_path(monkeypatch, tmp_path, python_path_on): +def test_python_path(monkeypatch, tmp_path, python_path_on) -> None: result = cli_run([str(tmp_path), "--without-pip", "--activators", ""]) monkeypatch.chdir(tmp_path) case_sensitive = fs_is_case_sensitive() @@ -675,12 +762,12 @@ def _get_sys_path(flag=None): # # https://github.com/pypa/virtualenv/issues/2419 @pytest.mark.skipif("venv" not in CURRENT_CREATORS, reason="test needs venv creator") -def test_venv_creator_without_write_perms(tmp_path, mocker): - from virtualenv.run.session import Session # noqa: PLC0415 +def test_venv_creator_without_write_perms(tmp_path, mocker) -> None: + from virtualenv.run.session import Session # ruff:ignore[import-outside-top-level] - prev = Session._create # noqa: SLF001 + prev = Session._create # ruff:ignore[private-member-access] - def func(self): + def func(self) -> None: prev(self) scripts_dir = self.creator.dest / "bin" for script in scripts_dir.glob("*ctivate*"): @@ -692,7 +779,7 @@ def func(self): cli_run(cmd) -def test_fallback_to_copies_if_symlink_unsupported(tmp_path, python, mocker): +def test_fallback_to_copies_if_symlink_unsupported(tmp_path, python, mocker) -> None: """Test that creating a virtual environment falls back to copies when filesystem has no symlink support.""" if is_macos_brew(PythonInfo.from_exe(python)): pytest.skip("brew python on darwin may not support copies, which is tested separately") @@ -717,7 +804,7 @@ def test_fallback_to_copies_if_symlink_unsupported(tmp_path, python, mocker): assert result.creator.symlinks is False -def test_fail_gracefully_if_no_method_supported(tmp_path, python, mocker): +def test_fail_gracefully_if_no_method_supported(tmp_path, python, mocker) -> None: """Test that virtualenv fails gracefully when no creation method is supported.""" # Given a filesystem that does not support symlinks mocker.patch("virtualenv.create.via_global_ref.api.fs_supports_symlink", return_value=False) @@ -726,7 +813,7 @@ def test_fail_gracefully_if_no_method_supported(tmp_path, python, mocker): if not is_macos_brew(PythonInfo.from_exe(python)): original_init = api.ViaGlobalRefMeta.__init__ - def new_init(self, *args, **kwargs): + def new_init(self, *args, **kwargs) -> None: original_init(self, *args, **kwargs) self.copy_error = "copying is not supported" @@ -746,7 +833,77 @@ def new_init(self, *args, **kwargs): # Then a RuntimeError should be raised with a detailed message assert "neither symlink or copy method supported" in str(excinfo.value) assert "symlink: the filesystem does not supports symlink" in str(excinfo.value) - if is_macos_brew(PythonInfo.from_exe(python)): + interpreter = PythonInfo.from_exe(python) + if is_macos_brew(interpreter): assert "copy: Brew disables copy creation" in str(excinfo.value) + elif is_mac_os_framework(interpreter): + assert "copy: macOS framework builds do not support copy-based virtual environments" in str(excinfo.value) else: assert "copy: copying is not supported" in str(excinfo.value) + + +def test_pyenv_cfg_preserves_symlinks(tmp_path) -> None: + """Test that PyEnvCfg.write() preserves symlinks and doesn't resolve them (issue #2770).""" + # Create a real directory and a symlink to it + real_dir = tmp_path / "real_directory" + real_dir.mkdir() + real_file = real_dir / "some_file.txt" + real_file.write_text("test", encoding="utf-8") + + symlink_dir = tmp_path / "symlink_directory" + try: + symlink_dir.symlink_to(real_dir, target_is_directory=True) + except (OSError, NotImplementedError): + pytest.skip("Symlinks not supported on this platform") + + cfg_path = tmp_path / "pyvenv.cfg" + cfg = PyEnvCfg(OrderedDict(), cfg_path) + + symlink_path = str(symlink_dir / "some_file.txt") + cfg["home"] = symlink_path + cfg.write() + + written_content = cfg_path.read_text(encoding="utf-8") + expected_abspath = os.path.abspath(symlink_path) + expected_realpath = os.path.realpath(symlink_path) + + assert f"home = {expected_abspath}" in written_content + assert expected_abspath != expected_realpath, "Test setup error: paths should differ for symlinks" + + +@pytest.mark.parametrize( + ("key", "value", "is_path"), + [ + pytest.param("home", "base", True, id="home"), + pytest.param("python-version", "3.14", False, id="python-version"), + pytest.param("version", "3.14.6", False, id="version"), + pytest.param("implementation", "CPython", False, id="implementation"), + pytest.param("include-system-site-packages", "true", False, id="include-system-site-packages"), + ], +) +def test_pyenv_cfg_value_naming_entry_in_cwd( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, key: str, value: str, is_path: bool +) -> None: + (tmp_path / value).mkdir() + monkeypatch.chdir(tmp_path) + cfg = PyEnvCfg(OrderedDict({key: value}), tmp_path / "pyvenv.cfg") + + cfg.write() + + assert PyEnvCfg.from_file(cfg.path).content == {key: os.path.abspath(value) if is_path else value} + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + pytest.param("\n\nhome = /a\n", {"home": "/a"}, id="blank-line"), + pytest.param("# home = /b\nhome = /a\n", {"home": "/a"}, id="comment"), + pytest.param(" # home = /b\nhome = /a\n", {"home": "/a"}, id="indented-comment"), + pytest.param("no-separator\nhome = /a\n", {"home": "/a"}, id="no-separator"), + pytest.param('home = /a\nprompt = "a b"\n', {"home": "/a", "prompt": "a b"}, id="quoted-value"), + ], +) +def test_pyenv_cfg_from_folder_skips_non_entries(tmp_path: Path, text: str, expected: dict[str, str]) -> None: + (tmp_path / "pyvenv.cfg").write_text(text, encoding="utf-8") + + assert PyEnvCfg.from_folder(tmp_path).content == expected diff --git a/tests/unit/create/test_interpreters.py b/tests/unit/create/test_interpreters.py index ae4452b13..73668ce2e 100644 --- a/tests/unit/create/test_interpreters.py +++ b/tests/unit/create/test_interpreters.py @@ -4,13 +4,13 @@ from uuid import uuid4 import pytest +from python_discovery import PythonInfo -from virtualenv.discovery.py_info import PythonInfo from virtualenv.run import cli_run @pytest.mark.slow -def test_failed_to_find_bad_spec(): +def test_failed_to_find_bad_spec() -> None: of_id = uuid4().hex with pytest.raises(RuntimeError) as context: cli_run(["-p", of_id]) @@ -25,7 +25,7 @@ def test_failed_to_find_bad_spec(): "of_id", ({sys.executable} if sys.executable != SYSTEM.executable else set()) | {SYSTEM.implementation}, ) -def test_failed_to_find_implementation(of_id, mocker): +def test_failed_to_find_implementation(of_id, mocker) -> None: mocker.patch("virtualenv.run.plugin.creators.CreatorSelector._OPTIONS", return_value={}) with pytest.raises(RuntimeError) as context: cli_run(["-p", of_id]) diff --git a/tests/unit/create/test_venv_redirect.py b/tests/unit/create/test_venv_redirect.py new file mode 100644 index 000000000..03dcef56a --- /dev/null +++ b/tests/unit/create/test_venv_redirect.py @@ -0,0 +1,149 @@ +from __future__ import annotations + +import logging +import os +from stat import S_IREAD, S_IWRITE +from typing import TYPE_CHECKING + +import pytest + +from virtualenv.run import cli_run + +if TYPE_CHECKING: + from pathlib import Path + + +@pytest.fixture +def redirect(tmp_path: Path) -> Path: + return tmp_path / ".venv" + + +@pytest.fixture +def project_redirect(redirect: Path) -> Path: + (redirect.parent / "pyproject.toml").write_text('[project]\nname = "demo"\n', encoding="utf-8") + return redirect + + +def _create(dest: Path, *args: str) -> None: + cli_run([str(dest), "--without-pip", "--activators", "", *args], setup_logging=False) + + +def test_venv_redirect_auto_skips_folder_without_pyproject(redirect: Path) -> None: + _create(redirect.parent / "env") + + assert not redirect.exists() + + +def test_venv_redirect_auto_keeps_existing_redirect(project_redirect: Path) -> None: + _create(project_redirect.parent / "old") + + _create(project_redirect.parent / "new") + + assert project_redirect.read_text(encoding="utf-8") == "old\n" + + +def test_venv_redirect_opted_out_in_project(project_redirect: Path) -> None: + _create(project_redirect.parent / "env", "--no-venv-redirect") + + assert not project_redirect.exists() + + +@pytest.mark.parametrize( + ("env_var", "env_value", "flags", "written"), + [ + pytest.param("VIRTUALENV_VENV_REDIRECT", "false", (), False, id="env-off"), + pytest.param("VIRTUALENV_NO_VENV_REDIRECT", "true", (), False, id="negative-env-off"), + pytest.param("VIRTUALENV_NO_VENV_REDIRECT", "false", (), True, id="negative-env-false-stays-auto"), + pytest.param("VIRTUALENV_NO_VENV_REDIRECT", "true", ("--venv-redirect",), True, id="cli-on-beats-env-off"), + pytest.param("VIRTUALENV_VENV_REDIRECT", "true", ("--no-venv-redirect",), False, id="cli-off-beats-env-on"), + ], +) +def test_venv_redirect_env_var_and_cli( # ruff:ignore[too-many-arguments] + project_redirect: Path, + monkeypatch: pytest.MonkeyPatch, + env_var: str, + env_value: str, + flags: tuple[str, ...], + written: bool, +) -> None: + monkeypatch.setenv(env_var, env_value) + + _create(project_redirect.parent / "env", *flags) + + assert project_redirect.exists() is written + + +def test_venv_redirect_forced_points_at_created(redirect: Path) -> None: + _create(redirect.parent / "env", "--venv-redirect") + + assert redirect.read_text(encoding="utf-8") == "env\n" + + +@pytest.mark.parametrize( + "line_end", [pytest.param("\n", id="lf"), pytest.param("\r\n", id="crlf"), pytest.param("", id="none")] +) +def test_venv_redirect_forced_moves_to_newest(redirect: Path, line_end: str) -> None: + _create(redirect.parent / "old", "--venv-redirect") + redirect.write_bytes(f"old{line_end}".encode()) + + _create(redirect.parent / "new", "--venv-redirect") + + assert redirect.read_text(encoding="utf-8") == "new\n" + + +@pytest.mark.parametrize( + "existing", + [ + pytest.param("missing\n", id="target-missing"), + pytest.param("", id="empty"), + ], +) +def test_venv_redirect_forced_keeps_foreign(redirect: Path, existing: str) -> None: + redirect.write_text(existing, encoding="utf-8") + + _create(redirect.parent / "env", "--venv-redirect") + + assert redirect.read_text(encoding="utf-8") == existing + + +def test_venv_redirect_forced_keeps_redirect_to_other_tool(redirect: Path) -> None: + other = redirect.parent / "other" + other.mkdir() + (other / "pyvenv.cfg").write_text("home = /usr/bin\nuv = 0.12.5\n", encoding="utf-8") + redirect.write_text("other\n", encoding="utf-8") + + _create(redirect.parent / "env", "--venv-redirect") + + assert redirect.read_text(encoding="utf-8") == "other\n" + + +def test_venv_redirect_forced_keeps_undecodable(redirect: Path) -> None: + redirect.write_bytes(b"\xff\xfeenv\n") + + _create(redirect.parent / "env", "--venv-redirect") + + assert redirect.read_bytes() == b"\xff\xfeenv\n" + + +def test_venv_redirect_forced_keeps_venv_folder(redirect: Path, caplog: pytest.LogCaptureFixture) -> None: + _create(redirect) + + with caplog.at_level(logging.DEBUG): + _create(redirect.parent / "env", "--venv-redirect") + + assert f"{redirect} keeps being the default environment" in caplog.text + + +def test_venv_redirect_forced_not_write_able(redirect: Path, caplog: pytest.LogCaptureFixture) -> None: + if hasattr(os, "geteuid") and os.geteuid() == 0: + pytest.skip("root may write read-only files") + _create(redirect.parent / "old", "--venv-redirect") + redirect.write_text("old\n", encoding="utf-8") + redirect.chmod(S_IREAD) + try: + with caplog.at_level(logging.WARNING): + _create(redirect.parent / "env", "--venv-redirect") + finally: + redirect.chmod(S_IREAD | S_IWRITE) + + assert f"could not point {redirect.resolve()} at {(redirect.parent / 'env').resolve()}" in caplog.text diff --git a/tests/unit/create/via_global_ref/_test_race_condition_helper.py b/tests/unit/create/via_global_ref/_test_race_condition_helper.py index 8027f17d3..02c75e9ac 100644 --- a/tests/unit/create/via_global_ref/_test_race_condition_helper.py +++ b/tests/unit/create/via_global_ref/_test_race_condition_helper.py @@ -7,7 +7,7 @@ class _Finder: fullname = None lock: ClassVar[list] = [] - def find_spec(self, fullname, path, target=None): # noqa: ARG002 + def find_spec(self, fullname, path, target=None) -> None: # ruff:ignore[unused-method-argument] # This should handle the NameError gracefully try: distutils_patch = _DISTUTILS_PATCH @@ -18,7 +18,7 @@ def find_spec(self, fullname, path, target=None): # noqa: ARG002 return @staticmethod - def exec_module(old, module): + def exec_module(old, module) -> None: old(module) try: distutils_patch = _DISTUTILS_PATCH @@ -27,16 +27,5 @@ def exec_module(old, module): if module.__name__ in distutils_patch: pass # Would call patch_dist(module) - @staticmethod - def load_module(old, name): - module = old(name) - try: - distutils_patch = _DISTUTILS_PATCH - except NameError: - return module - if module.__name__ in distutils_patch: - pass # Would call patch_dist(module) - return module - finder = _Finder() diff --git a/tests/unit/create/via_global_ref/builtin/conftest.py b/tests/unit/create/via_global_ref/builtin/conftest.py index a5808c58b..7d913fb96 100644 --- a/tests/unit/create/via_global_ref/builtin/conftest.py +++ b/tests/unit/create/via_global_ref/builtin/conftest.py @@ -2,11 +2,16 @@ import sys from pathlib import Path +from typing import TYPE_CHECKING +from unittest.mock import MagicMock import pytest from testing import path from testing.py_info import read_fixture +if TYPE_CHECKING: + from tests.types import Interpreter, MakeInterpreter + # Allows to import from `testing` into test submodules. sys.path.append(str(Path(__file__).parent)) @@ -24,3 +29,23 @@ def mock_files(mocker): @pytest.fixture def mock_pypy_libs(mocker): return lambda pypy, libs: path.mock_pypy_libs(mocker, pypy, libs) + + +@pytest.fixture +def make_interpreter() -> MakeInterpreter: + def _make( + sysconfig_vars: dict[str, object] | None = None, + prefix: str = "/usr", + free_threaded: bool = False, + version_info: tuple[int, ...] = (3, 14, 0), + ) -> Interpreter: + interpreter = MagicMock() + interpreter.prefix = prefix + interpreter.system_prefix = prefix + interpreter.system_executable = f"{prefix}/bin/python3" + interpreter.free_threaded = free_threaded + interpreter.version_info = MagicMock(major=version_info[0], minor=version_info[1]) + interpreter.sysconfig_vars = sysconfig_vars or {} + return interpreter + + return _make # type: ignore[return-value] diff --git a/tests/unit/create/via_global_ref/builtin/cpython/conftest.py b/tests/unit/create/via_global_ref/builtin/cpython/conftest.py new file mode 100644 index 000000000..9a3ce4ad2 --- /dev/null +++ b/tests/unit/create/via_global_ref/builtin/cpython/conftest.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +from virtualenv.create.via_global_ref.builtin.cpython.cpython3 import CPython3Posix + +if TYPE_CHECKING: + from collections.abc import Callable + from pathlib import Path + + from pytest_mock import MockerFixture + + from tests.types import MakeInterpreter + from virtualenv.create.via_global_ref.builtin.ref import PathRefToDest + + CollectSources = Callable[[dict[str, object]], list[PathRefToDest]] + + +@pytest.fixture +def collect_sources(tmp_path: Path, make_interpreter: MakeInterpreter, mocker: MockerFixture) -> CollectSources: + def _collect(sysconfig_vars: dict[str, object]) -> list[PathRefToDest]: + interpreter = make_interpreter( + sysconfig_vars={**sysconfig_vars, "PYTHONFRAMEWORK": ""}, + prefix=str(tmp_path), + ) + interpreter.system_executable = str(tmp_path / "bin" / "python3") + mocker.patch( + "virtualenv.create.via_global_ref.builtin.cpython.cpython3.Path.exists", + return_value=True, + ) + return list(CPython3Posix.sources(interpreter)) + + return _collect diff --git a/tests/unit/create/via_global_ref/builtin/cpython/cpython3_win_debug.json b/tests/unit/create/via_global_ref/builtin/cpython/cpython3_win_debug.json new file mode 100644 index 000000000..7143b21f5 --- /dev/null +++ b/tests/unit/create/via_global_ref/builtin/cpython/cpython3_win_debug.json @@ -0,0 +1,63 @@ +{ + "platform": "win32", + "implementation": "CPython", + "version_info": { + "major": 3, + "minor": 13, + "micro": 0, + "releaselevel": "final", + "serial": 0 + }, + "architecture": 64, + "version_nodot": "313", + "version": "3.13.0 (main, Oct 07 2024, 00:00:00) [MSC v.1941 64 bit (AMD64)]", + "os": "nt", + "prefix": "c:\\path\\to\\python", + "base_prefix": "c:\\path\\to\\python", + "real_prefix": null, + "base_exec_prefix": "c:\\path\\to\\python", + "exec_prefix": "c:\\path\\to\\python", + "executable": "c:\\path\\to\\python\\python_d.exe", + "original_executable": "c:\\path\\to\\python\\python_d.exe", + "system_executable": "c:\\path\\to\\python\\python_d.exe", + "has_venv": false, + "path": [ + "c:\\path\\to\\python\\Scripts\\virtualenv.exe", + "c:\\path\\to\\python\\python313.zip", + "c:\\path\\to\\python", + "c:\\path\\to\\python\\Lib\\site-packages" + ], + "file_system_encoding": "utf-8", + "stdout_encoding": "utf-8", + "sysconfig_scheme": null, + "sysconfig_paths": { + "stdlib": "{installed_base}/Lib", + "platstdlib": "{base}/Lib", + "purelib": "{base}/Lib/site-packages", + "platlib": "{base}/Lib/site-packages", + "include": "{installed_base}/Include", + "scripts": "{base}/Scripts", + "data": "{base}" + }, + "distutils_install": { + "purelib": "Lib\\site-packages", + "platlib": "Lib\\site-packages", + "headers": "Include\\UNKNOWN", + "scripts": "Scripts", + "data": "" + }, + "sysconfig": { + "makefile_filename": "c:\\path\\to\\python\\Lib\\config\\Makefile" + }, + "sysconfig_vars": { + "PYTHONFRAMEWORK": "", + "installed_base": "c:\\path\\to\\python", + "base": "c:\\path\\to\\python" + }, + "system_stdlib": "c:\\path\\to\\python\\Lib", + "system_stdlib_platform": "c:\\path\\to\\python\\Lib", + "max_size": 9223372036854775807, + "_creators": null, + "free_threaded": false, + "debug_build": true +} diff --git a/tests/unit/create/via_global_ref/builtin/cpython/cpython3_win_embed.json b/tests/unit/create/via_global_ref/builtin/cpython/cpython3_win_embed.json index c75c6f4fc..515f7b7a0 100644 --- a/tests/unit/create/via_global_ref/builtin/cpython/cpython3_win_embed.json +++ b/tests/unit/create/via_global_ref/builtin/cpython/cpython3_win_embed.json @@ -58,5 +58,6 @@ "system_stdlib_platform": "c:\\path\\to\\python\\Lib", "max_size": 9223372036854775807, "_creators": null, - "free_threaded": false + "free_threaded": false, + "debug_build": false } diff --git a/tests/unit/create/via_global_ref/builtin/cpython/cpython3_win_free_threaded.json b/tests/unit/create/via_global_ref/builtin/cpython/cpython3_win_free_threaded.json new file mode 100644 index 000000000..44130a075 --- /dev/null +++ b/tests/unit/create/via_global_ref/builtin/cpython/cpython3_win_free_threaded.json @@ -0,0 +1,63 @@ +{ + "platform": "win32", + "implementation": "CPython", + "version_info": { + "major": 3, + "minor": 13, + "micro": 0, + "releaselevel": "final", + "serial": 0 + }, + "architecture": 64, + "version_nodot": "313", + "version": "3.13.0 (main, Oct 07 2024, 00:00:00) [MSC v.1941 64 bit (AMD64)]", + "os": "nt", + "prefix": "c:\\path\\to\\python", + "base_prefix": "c:\\path\\to\\python", + "real_prefix": null, + "base_exec_prefix": "c:\\path\\to\\python", + "exec_prefix": "c:\\path\\to\\python", + "executable": "c:\\path\\to\\python\\python3.13t.exe", + "original_executable": "c:\\path\\to\\python\\python3.13t.exe", + "system_executable": "c:\\path\\to\\python\\python3.13t.exe", + "has_venv": false, + "path": [ + "c:\\path\\to\\python\\Scripts\\virtualenv.exe", + "c:\\path\\to\\python\\python313.zip", + "c:\\path\\to\\python", + "c:\\path\\to\\python\\Lib\\site-packages" + ], + "file_system_encoding": "utf-8", + "stdout_encoding": "utf-8", + "sysconfig_scheme": null, + "sysconfig_paths": { + "stdlib": "{installed_base}/Lib", + "platstdlib": "{base}/Lib", + "purelib": "{base}/Lib/site-packages", + "platlib": "{base}/Lib/site-packages", + "include": "{installed_base}/Include", + "scripts": "{base}/Scripts", + "data": "{base}" + }, + "distutils_install": { + "purelib": "Lib\\site-packages", + "platlib": "Lib\\site-packages", + "headers": "Include\\UNKNOWN", + "scripts": "Scripts", + "data": "" + }, + "sysconfig": { + "makefile_filename": "c:\\path\\to\\python\\Lib\\config\\Makefile" + }, + "sysconfig_vars": { + "PYTHONFRAMEWORK": "", + "installed_base": "c:\\path\\to\\python", + "base": "c:\\path\\to\\python" + }, + "system_stdlib": "c:\\path\\to\\python\\Lib", + "system_stdlib_platform": "c:\\path\\to\\python\\Lib", + "max_size": 9223372036854775807, + "_creators": null, + "free_threaded": true, + "debug_build": false +} diff --git a/tests/unit/create/via_global_ref/builtin/cpython/test_cpython3_posix.py b/tests/unit/create/via_global_ref/builtin/cpython/test_cpython3_posix.py new file mode 100644 index 000000000..53ced18b2 --- /dev/null +++ b/tests/unit/create/via_global_ref/builtin/cpython/test_cpython3_posix.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from virtualenv.create.via_global_ref.builtin.cpython.cpython3 import CPython3Posix +from virtualenv.create.via_global_ref.builtin.ref import PathRefToDest, RefWhen + +if TYPE_CHECKING: + from pathlib import Path + + from conftest import CollectSources + + from tests.types import MakeInterpreter + + +def _shared_lib_copy_refs(sources: list[PathRefToDest]) -> list[PathRefToDest]: + return [s for s in sources if isinstance(s, PathRefToDest) and s.when == RefWhen.COPY] + + +def test_shared_lib_included(tmp_path: Path, collect_sources: CollectSources) -> None: + lib_dir = tmp_path / "lib" + lib_dir.mkdir() + (lib_dir / "libpython3.14.so").touch() + sources = collect_sources({"Py_ENABLE_SHARED": 1, "INSTSONAME": "libpython3.14.so", "LIBDIR": str(lib_dir)}) + shared_refs = _shared_lib_copy_refs(sources) + assert len(shared_refs) == 1 + assert shared_refs[0].src.name == "libpython3.14.so" + + +def test_shared_lib_excluded_when_static(collect_sources: CollectSources) -> None: + sources = collect_sources({"Py_ENABLE_SHARED": 0}) + assert _shared_lib_copy_refs(sources) == [] + + +def test_shared_lib_excluded_when_no_lib_name(collect_sources: CollectSources) -> None: + sources = collect_sources({"Py_ENABLE_SHARED": 1}) + assert _shared_lib_copy_refs(sources) == [] + + +def test_shared_lib_excluded_when_no_lib_dir(collect_sources: CollectSources) -> None: + sources = collect_sources({"Py_ENABLE_SHARED": 1, "INSTSONAME": "libpython3.14.so"}) + assert _shared_lib_copy_refs(sources) == [] + + +def test_shared_lib_excluded_when_file_missing(tmp_path: Path, make_interpreter: MakeInterpreter) -> None: + lib_dir = tmp_path / "lib" + lib_dir.mkdir() + interpreter = make_interpreter( + sysconfig_vars={ + "Py_ENABLE_SHARED": 1, + "INSTSONAME": "libpython3.14.so", + "LIBDIR": str(lib_dir), + "PYTHONFRAMEWORK": "", + }, + prefix=str(tmp_path), + ) + interpreter.system_executable = str(tmp_path / "bin" / "python3") + sources = list(CPython3Posix.sources(interpreter)) + assert _shared_lib_copy_refs(sources) == [] diff --git a/tests/unit/create/via_global_ref/builtin/cpython/test_cpython3_win.py b/tests/unit/create/via_global_ref/builtin/cpython/test_cpython3_win.py index 4367ebc50..36663b729 100644 --- a/tests/unit/create/via_global_ref/builtin/cpython/test_cpython3_win.py +++ b/tests/unit/create/via_global_ref/builtin/cpython/test_cpython3_win.py @@ -1,7 +1,7 @@ from __future__ import annotations import pytest -from testing.helpers import contains_exe, contains_ref +from testing.helpers import contains_exe, contains_ref, has_src, is_exe from testing.path import join as path from virtualenv.create.via_global_ref.builtin.cpython.cpython3 import CPython3Windows @@ -13,7 +13,7 @@ @pytest.mark.parametrize("py_info_name", ["cpython3_win_embed"]) -def test_2_exe_on_default_py_host(py_info, mock_files): +def test_2_exe_on_default_py_host(py_info, mock_files) -> None: mock_files(CPYTHON3_PATH, [py_info.system_executable]) sources = tuple(CPython3Windows.sources(interpreter=py_info)) # Default Python exe. @@ -23,7 +23,7 @@ def test_2_exe_on_default_py_host(py_info, mock_files): @pytest.mark.parametrize("py_info_name", ["cpython3_win_embed"]) -def test_3_exe_on_not_default_py_host(py_info, mock_files): +def test_3_exe_on_not_default_py_host(py_info, mock_files) -> None: # Not default python host. py_info.system_executable = path(py_info.prefix, "python666.exe") mock_files(CPYTHON3_PATH, [py_info.system_executable]) @@ -36,7 +36,7 @@ def test_3_exe_on_not_default_py_host(py_info, mock_files): @pytest.mark.parametrize("py_info_name", ["cpython3_win_embed"]) -def test_only_shim(py_info, mock_files): +def test_only_shim(py_info, mock_files) -> None: shim = path(py_info.system_stdlib, "venv\\scripts\\nt\\python.exe") py_files = ( path(py_info.prefix, "libcrypto-1_1.dll"), @@ -54,7 +54,7 @@ def test_only_shim(py_info, mock_files): @pytest.mark.parametrize("py_info_name", ["cpython3_win_embed"]) -def test_exe_dll_pyd_without_shim(py_info, mock_files): +def test_exe_dll_pyd_without_shim(py_info, mock_files) -> None: py_files = ( path(py_info.prefix, "libcrypto-1_1.dll"), path(py_info.prefix, "libffi-7.dll"), @@ -70,7 +70,7 @@ def test_exe_dll_pyd_without_shim(py_info, mock_files): @pytest.mark.parametrize("py_info_name", ["cpython3_win_embed"]) -def test_python_zip_if_exists_and_set_in_path(py_info, mock_files): +def test_python_zip_if_exists_and_set_in_path(py_info, mock_files) -> None: python_zip_name = f"python{py_info.version_nodot}.zip" python_zip = path(py_info.prefix, python_zip_name) mock_files(CPYTHON3_PATH, [python_zip]) @@ -80,7 +80,7 @@ def test_python_zip_if_exists_and_set_in_path(py_info, mock_files): @pytest.mark.parametrize("py_info_name", ["cpython3_win_embed"]) -def test_no_python_zip_if_exists_and_not_set_in_path(py_info, mock_files): +def test_no_python_zip_if_exists_and_not_set_in_path(py_info, mock_files) -> None: python_zip_name = f"python{py_info.version_nodot}.zip" python_zip = path(py_info.prefix, python_zip_name) py_info.path.remove(python_zip) @@ -91,7 +91,7 @@ def test_no_python_zip_if_exists_and_not_set_in_path(py_info, mock_files): @pytest.mark.parametrize("py_info_name", ["cpython3_win_embed"]) -def test_no_python_zip_if_not_exists(py_info, mock_files): +def test_no_python_zip_if_not_exists(py_info, mock_files) -> None: python_zip_name = f"python{py_info.version_nodot}.zip" python_zip = path(py_info.prefix, python_zip_name) # No `python_zip`, just python.exe file. @@ -102,8 +102,108 @@ def test_no_python_zip_if_not_exists(py_info, mock_files): @pytest.mark.parametrize("py_info_name", ["cpython3_win_embed"]) -def test_python3_exe_present(py_info, mock_files): +def test_python3_exe_present(py_info, mock_files) -> None: mock_files(CPYTHON3_PATH, [py_info.system_executable]) sources = tuple(CPython3Windows.sources(interpreter=py_info)) assert contains_exe(sources, py_info.system_executable, "python3.exe") assert contains_exe(sources, py_info.system_executable, "python3") + + +@pytest.mark.parametrize("py_info_name", ["cpython3_win_embed"]) +def test_pythonw3_exe_present(py_info, mock_files) -> None: + mock_files(CPYTHON3_PATH, [py_info.system_executable]) + sources = tuple(CPython3Windows.sources(interpreter=py_info)) + pythonw_refs = [s for s in sources if is_exe(s) and has_src(path(py_info.prefix, "pythonw.exe"))(s)] + assert len(pythonw_refs) == 1 + assert "pythonw3.exe" in pythonw_refs[0].aliases + + +@pytest.mark.parametrize("py_info_name", ["cpython3_win_free_threaded"]) +def test_free_threaded_exe_naming(py_info, mock_files) -> None: + mock_files(CPYTHON3_PATH, [py_info.system_executable]) + sources = tuple(CPython3Windows.sources(interpreter=py_info)) + assert contains_exe(sources, py_info.system_executable, "python3.13t.exe") + pythonw_refs = [s for s in sources if is_exe(s) and has_src(path(py_info.prefix, "pythonw.exe"))(s)] + assert len(pythonw_refs) == 1 + assert "pythonw3.exe" in pythonw_refs[0].aliases + assert "pythonw3.13t.exe" in pythonw_refs[0].aliases + + +@pytest.mark.parametrize("py_info_name", ["cpython3_win_debug"]) +def test_debug_build_shim(py_info, mock_files) -> None: + shim = path(py_info.system_stdlib, "venv\\scripts\\nt\\venvlauncher_d.exe") + mock_files(CPYTHON3_PATH, [shim]) + assert CPython3Windows.has_shim(interpreter=py_info) + sources = tuple(CPython3Windows.sources(interpreter=py_info)) + assert contains_exe(sources, shim) + + +@pytest.mark.parametrize( + ("py_info_name", "shim_name"), + [ + pytest.param("cpython3_win_debug", "venvlauncher_d.exe", id="debug"), + pytest.param("cpython3_win_free_threaded", "venvlaunchert.exe", id="free-threaded"), + ], +) +def test_shim_not_copied_under_its_own_name(py_info, mock_files, shim_name: str) -> None: + shim = path(py_info.system_stdlib, f"venv\\scripts\\nt\\{shim_name}") + mock_files(CPYTHON3_PATH, [shim]) + sources = tuple(CPython3Windows.sources(interpreter=py_info)) + assert not contains_exe(sources, shim, shim_name) + + +@pytest.mark.parametrize( + ("py_info_name", "shim_name", "exe_name"), + [ + pytest.param("cpython3_win_debug", "venvlauncher_d.exe", "python_d.exe", id="3.13 debug build"), + pytest.param("cpython3_win_embed", "python.exe", "python666.exe", id="pre-3.13 renamed host"), + ], +) +def test_shim_exposes_the_interpreter_exe_name(py_info, mock_files, shim_name: str, exe_name: str) -> None: + py_info.system_executable = path(py_info.prefix, exe_name) + shim = path(py_info.system_stdlib, f"venv\\scripts\\nt\\{shim_name}") + mock_files(CPYTHON3_PATH, [shim]) + sources = tuple(CPython3Windows.sources(interpreter=py_info)) + assert contains_exe(sources, shim, exe_name) + + +@pytest.mark.parametrize("py_info_name", ["cpython3_win_debug"]) +def test_debug_build_uses_venvlauncher(py_info, mock_files) -> None: + launcher = path(py_info.prefix, "venvlauncher_d.exe") + w_launcher = path(py_info.prefix, "venvwlauncher_d.exe") + mock_files(CPYTHON3_PATH, [py_info.system_executable, launcher, w_launcher]) + sources = tuple(CPython3Windows.sources(interpreter=py_info)) + assert contains_exe(sources, launcher, "python.exe") + assert contains_exe(sources, w_launcher, "pythonw.exe") + + +@pytest.mark.parametrize("py_info_name", ["cpython3_win_embed"]) +def test_pywin32_dll_exclusion(py_info, mock_files) -> None: + """Test that pywin32 DLLs are excluded from virtualenv creation.""" + # Mock pywin32 DLLs that should be excluded + pywin32_dlls = ( + path(py_info.prefix, "pywintypes39.dll"), + path(py_info.prefix, "pywintypes310.dll"), + path(py_info.prefix, "pythoncom39.dll"), + path(py_info.prefix, "pythoncom310.dll"), + ) + # Mock regular DLLs that should be included + regular_dlls = ( + path(py_info.prefix, "libcrypto-1_1.dll"), + path(py_info.prefix, "libffi-7.dll"), + ) + # Only mock the DLL files (no shim) so dll_and_pyd() method will be called + all_files = [*pywin32_dlls, *regular_dlls] + mock_files(CPYTHON3_PATH, all_files) + sources = tuple(CPython3Windows.sources(interpreter=py_info)) + + # Make sure we're in the no-shim code path + assert not CPython3Windows.has_shim(interpreter=py_info) + + # Verify pywin32 DLLs are excluded + for dll in pywin32_dlls: + assert not contains_ref(sources, dll), f"pywin32 DLL {dll} should be excluded" + + # Verify regular DLLs are included + for dll in regular_dlls: + assert contains_ref(sources, dll), f"Regular DLL {dll} should be included" diff --git a/tests/unit/create/via_global_ref/builtin/pypy/test_pypy3.py b/tests/unit/create/via_global_ref/builtin/pypy/test_pypy3.py index 3ae905fdb..d63f36d62 100644 --- a/tests/unit/create/via_global_ref/builtin/pypy/test_pypy3.py +++ b/tests/unit/create/via_global_ref/builtin/pypy/test_pypy3.py @@ -1,10 +1,19 @@ from __future__ import annotations +from typing import TYPE_CHECKING +from unittest.mock import MagicMock + import pytest from testing.helpers import contains_exe, contains_ref from testing.path import join as path from virtualenv.create.via_global_ref.builtin.pypy.pypy3 import PyPy3Posix +from virtualenv.create.via_global_ref.builtin.ref import PathRefToDest + +if TYPE_CHECKING: + from pathlib import Path + + from pytest_mock import MockerFixture PYPY3_PATH = ( "virtualenv.create.via_global_ref.builtin.pypy.common.Path", @@ -15,7 +24,7 @@ # In `PyPy3Posix.sources()` `host_lib` will be broken in Python 2 for Windows, # so `py_file` will not be in sources. @pytest.mark.parametrize("py_info_name", ["portable_pypy38"]) -def test_portable_pypy3_virtualenvs_get_their_libs(py_info, mock_files, mock_pypy_libs): +def test_portable_pypy3_virtualenvs_get_their_libs(py_info, mock_files, mock_pypy_libs) -> None: py_file = path(py_info.prefix, "lib/libgdbm.so.4") mock_files(PYPY3_PATH, [py_info.system_executable, py_file]) lib_file = path(py_info.prefix, "bin/libpypy3-c.so") @@ -28,7 +37,7 @@ def test_portable_pypy3_virtualenvs_get_their_libs(py_info, mock_files, mock_pyp @pytest.mark.parametrize("py_info_name", ["deb_pypy37"]) -def test_debian_pypy37_virtualenvs(py_info, mock_files, mock_pypy_libs): +def test_debian_pypy37_virtualenvs(py_info, mock_files, mock_pypy_libs) -> None: # Debian's pypy3 layout, installed to /usr, before 3.8 allowed a /usr prefix mock_files(PYPY3_PATH, [py_info.system_executable]) lib_file = path(py_info.prefix, "bin/libpypy3-c.so") @@ -40,10 +49,36 @@ def test_debian_pypy37_virtualenvs(py_info, mock_files, mock_pypy_libs): @pytest.mark.parametrize("py_info_name", ["deb_pypy38"]) -def test_debian_pypy38_virtualenvs_exclude_usr(py_info, mock_files, mock_pypy_libs): +def test_debian_pypy38_virtualenvs_exclude_usr(py_info, mock_files, mock_pypy_libs) -> None: mock_files(PYPY3_PATH, [py_info.system_executable, "/usr/lib/foo"]) # libpypy3-c.so lives on the ld search path mock_pypy_libs(PyPy3Posix, []) sources = tuple(PyPy3Posix.sources(interpreter=py_info)) assert len(sources) == 1 assert contains_exe(sources, py_info.system_executable) + + +def test_pypy_portable_deps_txt(tmp_path: Path, mocker: MockerFixture) -> None: + host_lib = tmp_path / "lib" + host_lib.mkdir() + stdlib = host_lib / "pypy3.10" + stdlib.mkdir() + (host_lib / "libssl.so").touch() + (host_lib / "libcrypto.so").touch() + (host_lib / "unneeded.so").touch() + deps_file = host_lib / "PYPY_PORTABLE_DEPS.txt" + deps_file.write_text("libssl.so\nlibcrypto.so\n", encoding="utf-8") + + interpreter = MagicMock() + interpreter.system_prefix = str(tmp_path) + interpreter.system_executable = str(tmp_path / "bin" / "pypy3") + interpreter.system_stdlib = str(stdlib) + interpreter.version_info = MagicMock(major=3, minor=10) + + mocker.patch.object(PyPy3Posix, "_shared_libs", return_value=[]) + + sources = list(PyPy3Posix.sources(interpreter)) + ref_names = {s.src.name for s in sources if isinstance(s, PathRefToDest)} + assert "libssl.so" in ref_names + assert "libcrypto.so" in ref_names + assert "unneeded.so" not in ref_names diff --git a/tests/unit/create/via_global_ref/builtin/rustpython/rustpython_posix.json b/tests/unit/create/via_global_ref/builtin/rustpython/rustpython_posix.json new file mode 100644 index 000000000..23bde801b --- /dev/null +++ b/tests/unit/create/via_global_ref/builtin/rustpython/rustpython_posix.json @@ -0,0 +1,61 @@ +{ + "platform": "linux", + "implementation": "RustPython", + "version_info": { + "major": 3, + "minor": 14, + "micro": 0, + "releaselevel": "alpha", + "serial": 0 + }, + "architecture": 64, + "version": "3.14.0alpha (default, Nov 10 2025, 09:14:50)\n[RustPython 0.4.0 with rustc 1.83.0 (90b35a623 2024-11-26)]", + "os": "posix", + "prefix": "/usr/local", + "base_prefix": "/usr/local", + "real_prefix": null, + "base_exec_prefix": "/usr/local", + "exec_prefix": "/usr/local", + "executable": "/usr/local/bin/rustpython", + "original_executable": "/usr/local/bin/rustpython", + "system_executable": "/usr/local/bin/rustpython", + "has_venv": true, + "path": [ + "/usr/local/lib/python3.14", + "/usr/local/lib/python3.14/site-packages" + ], + "file_system_encoding": "utf-8", + "stdout_encoding": "UTF-8", + "sysconfig_scheme": null, + "sysconfig_paths": { + "stdlib": "{installed_base}/lib/python{py_version_short}", + "platstdlib": "{platbase}/lib/python{py_version_short}", + "purelib": "{base}/lib/python{py_version_short}/site-packages", + "platlib": "{platbase}/lib/python{py_version_short}/site-packages", + "include": "{installed_base}/include/python{py_version_short}{abiflags}", + "scripts": "{base}/bin", + "data": "{base}" + }, + "distutils_install": { + "purelib": "lib/python3.14/site-packages", + "platlib": "lib/python3.14/site-packages", + "headers": "include/python3.14/UNKNOWN", + "scripts": "bin", + "data": "" + }, + "sysconfig": {}, + "sysconfig_vars": { + "installed_base": "/usr/local", + "implementation_lower": "rustpython", + "py_version_short": "3.14", + "platbase": "/usr/local", + "base": "/usr/local", + "abiflags": "t", + "PYTHONFRAMEWORK": "" + }, + "system_stdlib": "/usr/local/lib/python3.14", + "system_stdlib_platform": "/usr/local/lib/python3.14", + "max_size": 9223372036854775807, + "_creators": null, + "free_threaded": false +} diff --git a/tests/unit/create/via_global_ref/builtin/rustpython/rustpython_windows.json b/tests/unit/create/via_global_ref/builtin/rustpython/rustpython_windows.json new file mode 100644 index 000000000..f87b6dcfe --- /dev/null +++ b/tests/unit/create/via_global_ref/builtin/rustpython/rustpython_windows.json @@ -0,0 +1,58 @@ +{ + "platform": "win32", + "implementation": "RustPython", + "version_info": { + "major": 3, + "minor": 14, + "micro": 0, + "releaselevel": "alpha", + "serial": 0 + }, + "architecture": 64, + "version": "3.14.0alpha (default, Nov 10 2025, 09:14:50)\n[RustPython 0.4.0 with rustc 1.83.0 (90b35a623 2024-11-26) MSC v.1929 64 bit (AMD64)]", + "os": "nt", + "prefix": "C:\\RustPython", + "base_prefix": "C:\\RustPython", + "real_prefix": null, + "base_exec_prefix": "C:\\RustPython", + "exec_prefix": "C:\\RustPython", + "executable": "C:\\RustPython\\rustpython.exe", + "original_executable": "C:\\RustPython\\rustpython.exe", + "system_executable": "C:\\RustPython\\rustpython.exe", + "has_venv": true, + "path": ["C:\\RustPython\\Lib", "C:\\RustPython\\Lib\\site-packages"], + "file_system_encoding": "utf-8", + "stdout_encoding": "UTF-8", + "sysconfig_scheme": null, + "sysconfig_paths": { + "stdlib": "{installed_base}\\Lib", + "platstdlib": "{platbase}\\Lib", + "purelib": "{base}\\Lib\\site-packages", + "platlib": "{platbase}\\Lib\\site-packages", + "include": "{installed_base}\\Include", + "scripts": "{base}\\Scripts", + "data": "{base}" + }, + "distutils_install": { + "purelib": "Lib\\site-packages", + "platlib": "Lib\\site-packages", + "headers": "Include\\UNKNOWN", + "scripts": "Scripts", + "data": "" + }, + "sysconfig": {}, + "sysconfig_vars": { + "installed_base": "C:\\RustPython", + "implementation_lower": "rustpython", + "py_version_short": "3.14", + "platbase": "C:\\RustPython", + "base": "C:\\RustPython", + "abiflags": "t", + "PYTHONFRAMEWORK": "" + }, + "system_stdlib": "C:\\RustPython\\Lib", + "system_stdlib_platform": "C:\\RustPython\\Lib", + "max_size": 9223372036854775807, + "_creators": null, + "free_threaded": false +} diff --git a/tests/unit/create/via_global_ref/builtin/rustpython/test_rustpython.py b/tests/unit/create/via_global_ref/builtin/rustpython/test_rustpython.py new file mode 100644 index 000000000..584b06945 --- /dev/null +++ b/tests/unit/create/via_global_ref/builtin/rustpython/test_rustpython.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from unittest.mock import MagicMock + +import pytest +from testing.helpers import contains_exe + +from virtualenv.create.via_global_ref.builtin.rustpython import RustPythonPosix, RustPythonWindows + + +@pytest.mark.parametrize("py_info_name", ["rustpython_posix"]) +def test_can_describe_rustpython_posix(py_info: MagicMock) -> None: + assert RustPythonPosix.can_describe(py_info) + + +@pytest.mark.parametrize("py_info_name", ["rustpython_windows"]) +def test_can_describe_rustpython_windows(py_info: MagicMock) -> None: + assert RustPythonWindows.can_describe(py_info) + + +def test_can_describe_rejects_cpython() -> None: + interpreter = MagicMock() + interpreter.implementation = "CPython" + interpreter.os = "posix" + assert not RustPythonPosix.can_describe(interpreter) + interpreter.os = "nt" + assert not RustPythonWindows.can_describe(interpreter) + + +@pytest.mark.parametrize("py_info_name", ["rustpython_posix"]) +def test_sources_posix(py_info: MagicMock, mock_files: object) -> None: + mock_files(("virtualenv.create.via_global_ref.builtin.rustpython.Path",), [py_info.system_executable]) + sources = list(RustPythonPosix.sources(interpreter=py_info)) + assert len(sources) == 1 + assert contains_exe(sources, py_info.system_executable) + + +@pytest.mark.parametrize("py_info_name", ["rustpython_windows"]) +def test_sources_windows(py_info: MagicMock, mock_files: object) -> None: + mock_files(("virtualenv.create.via_global_ref.builtin.rustpython.Path",), [py_info.system_executable]) + sources = list(RustPythonWindows.sources(interpreter=py_info)) + assert len(sources) == 1 + assert contains_exe(sources, py_info.system_executable) + + +def test_exe_names() -> None: + interpreter = MagicMock() + interpreter.version_info = MagicMock(major=3, minor=14) + names = RustPythonPosix.exe_names(interpreter) + assert names == {"rustpython", "python", "python3", "python3.14"} diff --git a/tests/unit/create/via_global_ref/builtin/test_ref.py b/tests/unit/create/via_global_ref/builtin/test_ref.py new file mode 100644 index 000000000..5e3e87dcc --- /dev/null +++ b/tests/unit/create/via_global_ref/builtin/test_ref.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +from pathlib import Path +from stat import S_IXGRP, S_IXOTH, S_IXUSR +from typing import TYPE_CHECKING + +import pytest +from python_discovery import PythonInfo + +from virtualenv.create.via_global_ref import api +from virtualenv.create.via_global_ref.builtin.ref import ExePathRefToDest, PathRef, PathRefToDest, RefWhen +from virtualenv.create.via_global_ref.builtin.via_global_self_do import ( + BuiltinViaGlobalRefMeta, + ViaGlobalRefVirtualenvBuiltin, +) + +if TYPE_CHECKING: + from collections.abc import Callable, Generator + + from pytest_mock import MockerFixture + + +@pytest.fixture +def exe_ref(mocker: MockerFixture, request: pytest.FixtureRequest) -> ExePathRefToDest: + """Mock the source so the mode is ours to pick - Windows only sets execute bits for .exe/.bat/.cmd/.com.""" + src = mocker.create_autospec(Path, instance=True) + src.stat.return_value = mocker.MagicMock(st_mode=request.param) # create_autospec drops dotted kwargs before 3.12 + return ExePathRefToDest(src, targets=["python"], dest=mocker.Mock()) + + +@pytest.mark.parametrize( + ("exe_ref", "expected"), + [ + pytest.param(0o644 | S_IXUSR, True, id="user"), + pytest.param(0o644 | S_IXGRP, True, id="group"), + pytest.param(0o644 | S_IXOTH, True, id="other"), + pytest.param(0o644, False, id="none"), + ], + indirect=["exe_ref"], +) +def test_can_run_honours_every_execute_bit(exe_ref: ExePathRefToDest, expected: bool) -> None: + assert exe_ref.can_run is expected + + +@pytest.mark.parametrize("exe_ref", [pytest.param(0o644, id="none")], indirect=True) +def test_can_run_is_cached(exe_ref: ExePathRefToDest) -> None: + assert exe_ref.can_run is False + assert exe_ref.can_run is False + assert exe_ref.src.stat.call_count == 1 + + +@pytest.fixture +def creator_meta(mocker: MockerFixture) -> Callable[[PathRef], BuiltinViaGlobalRefMeta]: + """Force symlink support on, otherwise a host without it starts with symlink_error already set.""" + mocker.patch.object(api, "fs_supports_symlink", return_value=True) + + def build(ref: PathRef) -> BuiltinViaGlobalRefMeta: + class Creator(ViaGlobalRefVirtualenvBuiltin): + @classmethod + def sources(cls, _interpreter: PythonInfo) -> Generator[PathRef]: + yield ref + + return Creator.can_create(mocker.create_autospec(PythonInfo, instance=True)) + + return build + + +@pytest.mark.parametrize( + ("when", "reported_on"), + [ + pytest.param(RefWhen.ANY, "error", id="any"), + pytest.param(RefWhen.COPY, "copy_error", id="copy"), + pytest.param(RefWhen.SYMLINK, "symlink_error", id="symlink"), + ], +) +def test_missing_source_is_reported_against_its_mode( + tmp_path: Path, creator_meta: Callable[[PathRef], BuiltinViaGlobalRefMeta], when: RefWhen, reported_on: str +) -> None: + ref = absent_ref(tmp_path, when) + assert getattr(creator_meta(ref), reported_on) == f"missing required file {ref}" + + +@pytest.mark.parametrize( + ("when", "collected"), + [ + pytest.param(RefWhen.ANY, False, id="any"), + pytest.param(RefWhen.COPY, True, id="copy"), + pytest.param(RefWhen.SYMLINK, True, id="symlink"), + ], +) +def test_missing_source_reaches_create_unless_both_modes_need_it( + tmp_path: Path, creator_meta: Callable[[PathRef], BuiltinViaGlobalRefMeta], when: RefWhen, collected: bool +) -> None: + ref = absent_ref(tmp_path, when) + assert creator_meta(ref).sources == ([ref] if collected else []) + + +def absent_ref(tmp_path: Path, when: RefWhen) -> PathRefToDest: + return PathRefToDest(tmp_path / "absent", dest=lambda _creator, src: src, when=when) diff --git a/tests/unit/create/via_global_ref/builtin/testing/path.py b/tests/unit/create/via_global_ref/builtin/testing/path.py index 06ba921c5..5ca60c5d6 100644 --- a/tests/unit/create/via_global_ref/builtin/testing/path.py +++ b/tests/unit/create/via_global_ref/builtin/testing/path.py @@ -75,20 +75,18 @@ def iterdir(self): yield from map(self.joinpath, self.contained_fake_names) -def MetaPathMock(filelist): # noqa: N802 - """ - Metaclass that creates a `PathMock` class with the `filelist` defined. - """ +def MetaPathMock(filelist): # ruff:ignore[invalid-function-name] + """Metaclass that creates a `PathMock` class with the `filelist` defined.""" return type("PathMock", (PathMockABC,), {"filelist": filelist}) -def mock_files(mocker, pathlist, filelist): - PathMock = MetaPathMock(set(filelist)) # noqa: N806 +def mock_files(mocker, pathlist, filelist) -> None: + PathMock = MetaPathMock(set(filelist)) # ruff:ignore[non-lowercase-variable-in-function] for path in pathlist: mocker.patch(path, PathMock) -def mock_pypy_libs(mocker, pypy_creator_cls, libs): +def mock_pypy_libs(mocker, pypy_creator_cls, libs) -> None: paths = tuple(set(map(Path, libs))) mocker.patch.object(pypy_creator_cls, "_shared_libs", return_value=paths) diff --git a/tests/unit/create/via_global_ref/builtin/testing/py_info.py b/tests/unit/create/via_global_ref/builtin/testing/py_info.py index 27a660c4a..99fc13e71 100644 --- a/tests/unit/create/via_global_ref/builtin/testing/py_info.py +++ b/tests/unit/create/via_global_ref/builtin/testing/py_info.py @@ -2,7 +2,7 @@ from pathlib import Path -from virtualenv.discovery.py_info import PythonInfo +from python_discovery import PythonInfo def fixture_file(fixture_name): @@ -18,4 +18,4 @@ def fixture_file(fixture_name): def read_fixture(fixture_name): fixture_json = fixture_file(fixture_name).read_text(encoding="utf-8") - return PythonInfo._from_json(fixture_json) # noqa: SLF001 + return PythonInfo.from_json(fixture_json) diff --git a/tests/unit/create/via_global_ref/test_api.py b/tests/unit/create/via_global_ref/test_api.py index a863b0e45..24c0afe45 100644 --- a/tests/unit/create/via_global_ref/test_api.py +++ b/tests/unit/create/via_global_ref/test_api.py @@ -1,8 +1,13 @@ from __future__ import annotations +from typing import TYPE_CHECKING + from virtualenv.create.via_global_ref import api +if TYPE_CHECKING: + from pytest_mock import MockerFixture + -def test_can_symlink_when_symlinks_not_enabled(mocker): +def test_can_symlink_when_symlinks_not_enabled(mocker: MockerFixture) -> None: mocker.patch.object(api, "fs_supports_symlink", return_value=False) assert api.ViaGlobalRefMeta().can_symlink is False diff --git a/tests/unit/create/via_global_ref/test_build_c_ext.py b/tests/unit/create/via_global_ref/test_build_c_ext.py index 5195ff856..d9347b794 100644 --- a/tests/unit/create/via_global_ref/test_build_c_ext.py +++ b/tests/unit/create/via_global_ref/test_build_c_ext.py @@ -4,72 +4,76 @@ import shutil import subprocess from pathlib import Path -from subprocess import Popen +from typing import TYPE_CHECKING, Final import pytest +from python_discovery import PythonInfo -from virtualenv.discovery.py_info import PythonInfo from virtualenv.run import cli_run +from virtualenv.run.plugin.creators import CreatorSelector +from virtualenv.seed.wheels.embed import BUNDLE_FOLDER as EMBED_WHEEL_DIR -CURRENT = PythonInfo.current_system() -CREATOR_CLASSES = CURRENT.creators().key_to_class +if TYPE_CHECKING: + from collections.abc import Callable + from virtualenv.create.creator import Creator -def builtin_shows_marker_missing(): - builtin_classs = CREATOR_CLASSES.get("builtin") - if builtin_classs is None: +_CURRENT: Final[PythonInfo] = PythonInfo.current_system() +_CREATOR_CLASSES: Final[dict[str, type[Creator]]] = CreatorSelector.for_interpreter(_CURRENT).key_to_class + + +def builtin_shows_marker_missing() -> bool: + if (builtin_class := _CREATOR_CLASSES.get("builtin")) is None: return False - host_include_marker = getattr(builtin_classs, "host_include_marker", None) - if host_include_marker is None: + if (host_include_marker := getattr(builtin_class, "host_include_marker", None)) is None: return False - marker = host_include_marker(CURRENT) - return not marker.exists() + return not host_include_marker(_CURRENT).exists() @pytest.mark.slow +@pytest.mark.timeout(300) @pytest.mark.xfail( condition=bool(os.environ.get("CI_RUN")), strict=False, reason="did not manage to setup CI to run with VC 14.1 C++ compiler, but passes locally", ) @pytest.mark.skipif( - not Path(CURRENT.system_include).exists() and not builtin_shows_marker_missing(), + not Path(_CURRENT.system_include).exists() and not builtin_shows_marker_missing(), reason="Building C-Extensions requires header files with host python", ) -@pytest.mark.parametrize("creator", [i for i in CREATOR_CLASSES if i != "builtin"]) -def test_can_build_c_extensions(creator, tmp_path, coverage_env): - env, greet = tmp_path / "env", str(tmp_path / "greet") +@pytest.mark.parametrize("creator", [i for i in _CREATOR_CLASSES if i != "builtin"]) +def test_can_build_c_extensions(creator: str, tmp_path: Path, coverage_env: Callable[[], None]) -> None: + greet = str(tmp_path / "greet") shutil.copytree(str(Path(__file__).parent.resolve() / "greet"), greet) - session = cli_run(["--creator", creator, "--seeder", "app-data", str(env), "-vvv"]) + session = cli_run(["--creator", creator, "--seeder", "app-data", str(tmp_path / "env"), "-vvv"]) coverage_env() - setuptools_index_args = () - if CURRENT.version_info >= (3, 12): - # requires to be able to install setuptools as build dependency - setuptools_index_args = ( - "--find-links", - "https://pypi.org/simple/setuptools/", - ) - cmd = [ - str(session.creator.script("pip")), + str(session.creator.exe), + "-m", + "pip", "install", "--no-index", - *setuptools_index_args, + "--find-links", + str(EMBED_WHEEL_DIR), "--no-deps", "--disable-pip-version-check", "-vvv", greet, ] - process = Popen(cmd) - process.communicate() - assert process.returncode == 0 + # xdist workers on Windows send child output to NUL and pytest-timeout can only os._exit() them there, so bound + # the build here and keep its log: a stall then fails this test with the output instead of killing the worker + with (log := tmp_path / "pip.log").open("w", encoding="utf-8") as stream: + try: + build = subprocess.run(cmd, stdout=stream, stderr=subprocess.STDOUT, timeout=180, check=False) + except subprocess.TimeoutExpired: + pytest.fail(f"pip install did not finish in 180s:\n{log.read_text(encoding='utf-8')}") + assert build.returncode == 0, log.read_text(encoding="utf-8") - process = Popen( + result = subprocess.run( [str(session.creator.exe), "-c", "import greet; greet.greet('World')"], - universal_newlines=True, - stdout=subprocess.PIPE, + capture_output=True, encoding="utf-8", + timeout=60, + check=False, ) - out, _ = process.communicate() - assert process.returncode == 0 - assert out == "Hello World!\n" + assert (result.returncode, result.stdout) == (0, "Hello World!\n"), result.stderr diff --git a/tests/unit/create/via_global_ref/test_race_condition.py b/tests/unit/create/via_global_ref/test_race_condition.py index 3b044167c..a445c2ce4 100644 --- a/tests/unit/create/via_global_ref/test_race_condition.py +++ b/tests/unit/create/via_global_ref/test_race_condition.py @@ -4,7 +4,24 @@ from pathlib import Path -def test_virtualenv_py_race_condition_find_spec(tmp_path): +def _test_finder_behavior() -> None: + import _virtualenv_test # ruff:ignore[import-outside-top-level] + + finder = _virtualenv_test.finder + + result = finder.find_spec("distutils.dist", None) + assert result is None, "find_spec should return None when _DISTUTILS_PATCH is not defined" + + class MockModule: + __name__ = "distutils.dist" + + def mock_old_exec(_x) -> None: + pass + + finder.exec_module(mock_old_exec, MockModule()) + + +def test_virtualenv_py_race_condition_find_spec(tmp_path) -> None: """Test that _Finder.find_spec handles NameError gracefully when _DISTUTILS_PATCH is not defined.""" # Create a temporary file with partial _virtualenv.py content (simulating race condition) venv_file = tmp_path / "_virtualenv_test.py" @@ -18,56 +35,7 @@ def test_virtualenv_py_race_condition_find_spec(tmp_path): sys.path.insert(0, str(tmp_path)) try: - import _virtualenv_test # noqa: PLC0415 - - finder = _virtualenv_test.finder - - # Try to call find_spec - this should not raise NameError - result = finder.find_spec("distutils.dist", None) - assert result is None, "find_spec should return None when _DISTUTILS_PATCH is not defined" - - # Create a mock module object - class MockModule: - __name__ = "distutils.dist" - - # Try to call exec_module - this should not raise NameError - def mock_old_exec(_x): - pass - - finder.exec_module(mock_old_exec, MockModule()) - - # Try to call load_module - this should not raise NameError - def mock_old_load(_name): - return MockModule() - - result = finder.load_module(mock_old_load, "distutils.dist") - assert result.__name__ == "distutils.dist" - + _test_finder_behavior() finally: sys.path.remove(str(tmp_path)) - if "_virtualenv_test" in sys.modules: - del sys.modules["_virtualenv_test"] - - -def test_virtualenv_py_normal_operation(): - """Test that the fix doesn't break normal operation when _DISTUTILS_PATCH is defined.""" - # Read the actual _virtualenv.py file - virtualenv_py_path = ( - Path(__file__).parent.parent.parent.parent.parent - / "src" - / "virtualenv" - / "create" - / "via_global_ref" - / "_virtualenv.py" - ) - - if not virtualenv_py_path.exists(): - return # Skip if we can't find the file - - content = virtualenv_py_path.read_text(encoding="utf-8") - - # Verify the fix is present - assert "try:" in content - assert "distutils_patch = _DISTUTILS_PATCH" in content - assert "except NameError:" in content - assert "return None" in content or "return" in content + sys.modules.pop("_virtualenv_test", None) diff --git a/tests/unit/discovery/py_info/test_py_info.py b/tests/unit/discovery/py_info/test_py_info.py deleted file mode 100644 index 15472daab..000000000 --- a/tests/unit/discovery/py_info/test_py_info.py +++ /dev/null @@ -1,531 +0,0 @@ -from __future__ import annotations - -import copy -import functools -import itertools -import json -import logging -import os -import sys -import sysconfig -from pathlib import Path -from textwrap import dedent -from typing import NamedTuple - -import pytest - -from virtualenv.create.via_global_ref.builtin.cpython.common import is_macos_brew -from virtualenv.discovery import cached_py_info -from virtualenv.discovery.py_info import PythonInfo, VersionInfo -from virtualenv.discovery.py_spec import PythonSpec -from virtualenv.info import IS_PYPY, IS_WIN, fs_supports_symlink - -CURRENT = PythonInfo.current_system() - - -def test_current_as_json(): - result = CURRENT._to_json() # noqa: SLF001 - parsed = json.loads(result) - a, b, c, d, e = sys.version_info - f = sysconfig.get_config_var("Py_GIL_DISABLED") == 1 - assert parsed["version_info"] == {"major": a, "minor": b, "micro": c, "releaselevel": d, "serial": e} - assert parsed["free_threaded"] is f - - -def test_bad_exe_py_info_raise(tmp_path, session_app_data): - exe = str(tmp_path) - with pytest.raises(RuntimeError) as context: - PythonInfo.from_exe(exe, session_app_data) - msg = str(context.value) - assert "code" in msg - assert exe in msg - - -def test_bad_exe_py_info_no_raise(tmp_path, caplog, capsys, session_app_data): - caplog.set_level(logging.NOTSET) - exe = str(tmp_path) - result = PythonInfo.from_exe(exe, session_app_data, raise_on_error=False) - assert result is None - out, _ = capsys.readouterr() - assert not out - messages = [r.message for r in caplog.records if r.name != "filelock"] - assert len(messages) == 2 - msg = messages[0] - assert "get interpreter info via cmd: " in msg - msg = messages[1] - assert str(exe) in msg - assert "code" in msg - - -@pytest.mark.parametrize( - "spec", - itertools.chain( - [sys.executable], - [ - f"{impl}{'.'.join(str(i) for i in ver)}{'t' if CURRENT.free_threaded else ''}{arch}" - for impl, ver, arch in itertools.product( - ( - [CURRENT.implementation] - + (["python"] if CURRENT.implementation == "CPython" else []) - + ( - [CURRENT.implementation.lower()] - if CURRENT.implementation != CURRENT.implementation.lower() - else [] - ) - ), - [sys.version_info[0 : i + 1] for i in range(3)], - ["", f"-{CURRENT.architecture}"], - ) - ], - ), -) -def test_satisfy_py_info(spec): - parsed_spec = PythonSpec.from_string_spec(spec) - matches = CURRENT.satisfies(parsed_spec, True) - assert matches is True - - -def test_satisfy_not_arch(): - parsed_spec = PythonSpec.from_string_spec( - f"{CURRENT.implementation}-{64 if CURRENT.architecture == 32 else 32}", - ) - matches = CURRENT.satisfies(parsed_spec, True) - assert matches is False - - -def test_satisfy_not_threaded(): - parsed_spec = PythonSpec.from_string_spec( - f"{CURRENT.implementation}{CURRENT.version_info.major}{'' if CURRENT.free_threaded else 't'}", - ) - matches = CURRENT.satisfies(parsed_spec, True) - assert matches is False - - -def _generate_not_match_current_interpreter_version(): - result = [] - for i in range(3): - ver = sys.version_info[0 : i + 1] - for a in range(len(ver)): - for o in [-1, 1]: - temp = list(ver) - temp[a] += o - result.append(".".join(str(i) for i in temp)) - return result - - -_NON_MATCH_VER = _generate_not_match_current_interpreter_version() - - -@pytest.mark.parametrize("spec", _NON_MATCH_VER) -def test_satisfy_not_version(spec): - parsed_spec = PythonSpec.from_string_spec(f"{CURRENT.implementation}{spec}") - matches = CURRENT.satisfies(parsed_spec, True) - assert matches is False - - -def test_py_info_cached_error(mocker, tmp_path, session_app_data): - spy = mocker.spy(cached_py_info, "_run_subprocess") - with pytest.raises(RuntimeError): - PythonInfo.from_exe(str(tmp_path), session_app_data) - with pytest.raises(RuntimeError): - PythonInfo.from_exe(str(tmp_path), session_app_data) - assert spy.call_count == 1 - - -@pytest.mark.skipif(not fs_supports_symlink(), reason="symlink is not supported") -def test_py_info_cached_symlink_error(mocker, tmp_path, session_app_data): - spy = mocker.spy(cached_py_info, "_run_subprocess") - with pytest.raises(RuntimeError): - PythonInfo.from_exe(str(tmp_path), session_app_data) - symlinked = tmp_path / "a" - symlinked.symlink_to(tmp_path) - with pytest.raises(RuntimeError): - PythonInfo.from_exe(str(symlinked), session_app_data) - assert spy.call_count == 2 - - -def test_py_info_cache_clear(mocker, session_app_data): - spy = mocker.spy(cached_py_info, "_run_subprocess") - result = PythonInfo.from_exe(sys.executable, session_app_data) - assert result is not None - count = 1 if result.executable == sys.executable else 2 # at least two, one for the venv, one more for the host - assert spy.call_count >= count - PythonInfo.clear_cache(session_app_data) - assert PythonInfo.from_exe(sys.executable, session_app_data) is not None - assert spy.call_count >= 2 * count - - -def test_py_info_cache_invalidation_on_py_info_change(mocker, session_app_data): - # 1. Get a PythonInfo object for the current executable, this will cache it. - PythonInfo.from_exe(sys.executable, session_app_data) - - # 2. Spy on _run_subprocess - spy = mocker.spy(cached_py_info, "_run_subprocess") - - # 3. Backup py_info.py - py_info_script = Path(cached_py_info.__file__).parent / "py_info.py" - original_content = py_info_script.read_text(encoding="utf-8") - original_stat = py_info_script.stat() - - try: - # 4. Clear the in-memory cache - mocker.patch.dict(cached_py_info._CACHE, {}, clear=True) # noqa: SLF001 - - # 5. Modify py_info.py to invalidate the cache - py_info_script.write_text(original_content + "\n# a comment", encoding="utf-8") - - # 6. Get the PythonInfo object again - info = PythonInfo.from_exe(sys.executable, session_app_data) - - # 7. Assert that _run_subprocess was called again - native_difference = 1 if info.system_executable == info.executable else 0 - if is_macos_brew(info): - assert spy.call_count + native_difference in {2, 3} - else: - assert spy.call_count + native_difference == 2 - - finally: - # 8. Restore the original content and timestamp - py_info_script.write_text(original_content, encoding="utf-8") - os.utime(str(py_info_script), (original_stat.st_atime, original_stat.st_mtime)) - - -@pytest.mark.skipif(not fs_supports_symlink(), reason="symlink is not supported") -@pytest.mark.xfail( - # https://doc.pypy.org/en/latest/install.html?highlight=symlink#download-a-pre-built-pypy - IS_PYPY and IS_WIN and sys.version_info[0:2] >= (3, 9), - reason="symlink is not supported", -) -@pytest.mark.skipif(not fs_supports_symlink(), reason="symlink is not supported") -def test_py_info_cached_symlink(mocker, tmp_path, session_app_data): - spy = mocker.spy(cached_py_info, "_run_subprocess") - first_result = PythonInfo.from_exe(sys.executable, session_app_data) - assert first_result is not None - count = spy.call_count - # at least two, one for the venv, one more for the host - exp_count = 1 if first_result.executable == sys.executable else 2 - assert count >= exp_count # at least two, one for the venv, one more for the host - - new_exe = tmp_path / "a" - new_exe.symlink_to(sys.executable) - pyvenv = Path(sys.executable).parents[1] / "pyvenv.cfg" - if pyvenv.exists(): - (tmp_path / pyvenv.name).write_text(pyvenv.read_text(encoding="utf-8"), encoding="utf-8") - new_exe_str = str(new_exe) - second_result = PythonInfo.from_exe(new_exe_str, session_app_data) - assert second_result.executable == new_exe_str - assert spy.call_count == count + 1 # no longer needed the host invocation, but the new symlink is must - - -class PyInfoMock(NamedTuple): - implementation: str - architecture: int - version_info: VersionInfo - - -@pytest.mark.parametrize( - ("target", "position", "discovered"), - [ - ( - PyInfoMock("CPython", 64, VersionInfo(3, 6, 8, "final", 0)), - 0, - [ - PyInfoMock("CPython", 64, VersionInfo(3, 6, 9, "final", 0)), - PyInfoMock("PyPy", 64, VersionInfo(3, 6, 8, "final", 0)), - ], - ), - ( - PyInfoMock("CPython", 64, VersionInfo(3, 6, 8, "final", 0)), - 0, - [ - PyInfoMock("CPython", 64, VersionInfo(3, 6, 9, "final", 0)), - PyInfoMock("CPython", 32, VersionInfo(3, 6, 9, "final", 0)), - ], - ), - ( - PyInfoMock("CPython", 64, VersionInfo(3, 8, 1, "final", 0)), - 0, - [ - PyInfoMock("CPython", 32, VersionInfo(2, 7, 12, "rc", 2)), - PyInfoMock("PyPy", 64, VersionInfo(3, 8, 1, "final", 0)), - ], - ), - ], -) -def test_system_executable_no_exact_match( # noqa: PLR0913 - target, - discovered, - position, - tmp_path, - mocker, - caplog, - session_app_data, -): - """Here we should fallback to other compatible""" - caplog.set_level(logging.DEBUG) - - def _make_py_info(of): - base = copy.deepcopy(CURRENT) - base.implementation = of.implementation - base.version_info = of.version_info - base.architecture = of.architecture - return base - - discovered_with_path = {} - names = [] - selected = None - for pos, i in enumerate(discovered): - path = tmp_path / str(pos) - path.write_text("", encoding="utf-8") - py_info = _make_py_info(i) - py_info.system_executable = CURRENT.system_executable - py_info.executable = CURRENT.system_executable - py_info.base_executable = str(path) - if pos == position: - selected = py_info - discovered_with_path[str(path)] = py_info - names.append(path.name) - - target_py_info = _make_py_info(target) - mocker.patch.object(target_py_info, "_find_possible_exe_names", return_value=names) - mocker.patch.object(target_py_info, "_find_possible_folders", return_value=[str(tmp_path)]) - - def func(k, app_data, resolve_to_host, raise_on_error, env): # noqa: ARG001 - return discovered_with_path[k] - - mocker.patch.object(target_py_info, "from_exe", side_effect=func) - target_py_info.real_prefix = str(tmp_path) - - target_py_info.system_executable = None - target_py_info.executable = str(tmp_path) - mapped = target_py_info._resolve_to_system(session_app_data, target_py_info) # noqa: SLF001 - assert mapped.system_executable == CURRENT.system_executable - found = discovered_with_path[mapped.base_executable] - assert found is selected - - assert caplog.records[0].msg == "discover exe for %s in %s" - for record in caplog.records[1:-1]: - assert record.message.startswith("refused interpreter ") - assert record.levelno == logging.DEBUG - - warn_similar = caplog.records[-1] - assert warn_similar.levelno == logging.DEBUG - assert warn_similar.msg.startswith("no exact match found, chosen most similar") - - -def test_py_info_ignores_distutils_config(monkeypatch, tmp_path): - raw = f""" - [install] - prefix={tmp_path}{os.sep}prefix - install_purelib={tmp_path}{os.sep}purelib - install_platlib={tmp_path}{os.sep}platlib - install_headers={tmp_path}{os.sep}headers - install_scripts={tmp_path}{os.sep}scripts - install_data={tmp_path}{os.sep}data - """ - (tmp_path / "setup.cfg").write_text(dedent(raw), encoding="utf-8") - monkeypatch.chdir(tmp_path) - py_info = PythonInfo.from_exe(sys.executable) - distutils = py_info.distutils_install - for key, value in distutils.items(): - assert not value.startswith(str(tmp_path)), f"{key}={value}" - - -def test_discover_exe_on_path_non_spec_name_match(mocker): - suffixed_name = f"python{CURRENT.version_info.major}.{CURRENT.version_info.minor}m" - if sys.platform == "win32": - suffixed_name += Path(CURRENT.original_executable).suffix - spec = PythonSpec.from_string_spec(suffixed_name) - mocker.patch.object(CURRENT, "original_executable", str(Path(CURRENT.executable).parent / suffixed_name)) - assert CURRENT.satisfies(spec, impl_must_match=True) is True - - -def test_discover_exe_on_path_non_spec_name_not_match(mocker): - suffixed_name = f"python{CURRENT.version_info.major}.{CURRENT.version_info.minor}m" - if sys.platform == "win32": - suffixed_name += Path(CURRENT.original_executable).suffix - spec = PythonSpec.from_string_spec(suffixed_name) - mocker.patch.object( - CURRENT, - "original_executable", - str(Path(CURRENT.executable).parent / f"e{suffixed_name}"), - ) - assert CURRENT.satisfies(spec, impl_must_match=True) is False - - -@pytest.mark.skipif(IS_PYPY, reason="setuptools distutils patching does not work") -def test_py_info_setuptools(): - from setuptools.dist import Distribution # noqa: PLC0415 - - assert Distribution - PythonInfo() - - -@pytest.mark.usefixtures("_skip_if_test_in_system") -def test_py_info_to_system_raises(session_app_data, mocker, caplog): - caplog.set_level(logging.DEBUG) - mocker.patch.object(PythonInfo, "_find_possible_folders", return_value=[]) - result = PythonInfo.from_exe(sys.executable, app_data=session_app_data, raise_on_error=False) - assert result is None - log = caplog.records[-1] - assert log.levelno == logging.INFO - expected = f"ignore {sys.executable} due cannot resolve system due to RuntimeError('failed to detect " - assert expected in log.message - - -def _stringify_schemes_dict(schemes_dict): - """ - Since this file has from __future__ import unicode_literals, we manually cast all values of mocked install_schemes - to str() as the original schemes are not unicode on Python 2. - """ - return {str(n): {str(k): str(v) for k, v in s.items()} for n, s in schemes_dict.items()} - - -def test_custom_venv_install_scheme_is_prefered(mocker): - # The paths in this test are Fedora paths, but we set them for nt as well, so the test also works on Windows, - # despite the actual values are nonsense there. - # Values were simplified to be compatible with all the supported Python versions. - default_scheme = { - "stdlib": "{base}/lib/python{py_version_short}", - "platstdlib": "{platbase}/lib/python{py_version_short}", - "purelib": "{base}/local/lib/python{py_version_short}/site-packages", - "platlib": "{platbase}/local/lib/python{py_version_short}/site-packages", - "include": "{base}/include/python{py_version_short}", - "platinclude": "{platbase}/include/python{py_version_short}", - "scripts": "{base}/local/bin", - "data": "{base}/local", - } - venv_scheme = {key: path.replace("local", "") for key, path in default_scheme.items()} - sysconfig_install_schemes = { - "posix_prefix": default_scheme, - "nt": default_scheme, - "pypy": default_scheme, - "pypy_nt": default_scheme, - "venv": venv_scheme, - } - if getattr(sysconfig, "get_preferred_scheme", None): - # define the prefix as sysconfig.get_preferred_scheme did before 3.11 - sysconfig_install_schemes["nt" if os.name == "nt" else "posix_prefix"] = default_scheme - - # On Python < 3.10, the distutils schemes are not derived from sysconfig schemes - # So we mock them as well to assert the custom "venv" install scheme has priority - distutils_scheme = { - "purelib": "$base/local/lib/python$py_version_short/site-packages", - "platlib": "$platbase/local/lib/python$py_version_short/site-packages", - "headers": "$base/include/python$py_version_short/$dist_name", - "scripts": "$base/local/bin", - "data": "$base/local", - } - distutils_schemes = { - "unix_prefix": distutils_scheme, - "nt": distutils_scheme, - } - - # We need to mock distutils first, so they don't see the mocked sysconfig, - # if imported for the first time. - # That can happen if the actual interpreter has the "venv" INSTALL_SCHEME - # and hence this is the first time we are touching distutils in this process. - # If distutils saw our mocked sysconfig INSTALL_SCHEMES, we would need - # to define all install schemes. - mocker.patch("distutils.command.install.INSTALL_SCHEMES", distutils_schemes) - mocker.patch("sysconfig._INSTALL_SCHEMES", sysconfig_install_schemes) - - pyinfo = PythonInfo() - pyver = f"{pyinfo.version_info.major}.{pyinfo.version_info.minor}" - assert pyinfo.install_path("scripts") == "bin" - assert pyinfo.install_path("purelib").replace(os.sep, "/") == f"lib/python{pyver}/site-packages" - - -@pytest.mark.skipif( - IS_PYPY or not (os.name == "posix" and sys.version_info[:2] >= (3, 11)), reason="POSIX 3.11+ specific" -) -def test_fallback_existent_system_executable(mocker): - current = PythonInfo() - # Posix may execute a "python" out of a venv but try to set the base_executable - # to "python" out of the system installation path. PEP 394 informs distributions - # that "python" is not required and the standard `make install` does not provide one - - # Falsify some data to look like we're in a venv - current.prefix = current.exec_prefix = "/tmp/tmp.izZNCyINRj/venv" # noqa: S108 - current.executable = current.original_executable = os.path.join(current.prefix, "bin/python") - - # Since we don't know if the distribution we're on provides python, use a binary that should not exist - mocker.patch.object(sys, "_base_executable", os.path.join(os.path.dirname(current.system_executable), "idontexist")) - mocker.patch.object(sys, "executable", current.executable) - - # ensure it falls back to an alternate binary name that exists - system_executable = current._fast_get_system_executable() # noqa: SLF001 - assert os.path.basename(system_executable) in [ - f"python{v}" for v in (current.version_info.major, f"{current.version_info.major}.{current.version_info.minor}") - ] - assert os.path.exists(system_executable) - - -@pytest.mark.skipif(sys.version_info[:2] != (3, 10), reason="3.10 specific") -def test_uses_posix_prefix_on_debian_3_10_without_venv(mocker): - # this is taken from ubuntu 22.04 /usr/lib/python3.10/sysconfig.py - sysconfig_install_schemes = { - "posix_prefix": { - "stdlib": "{installed_base}/{platlibdir}/python{py_version_short}", - "platstdlib": "{platbase}/{platlibdir}/python{py_version_short}", - "purelib": "{base}/lib/python{py_version_short}/site-packages", - "platlib": "{platbase}/{platlibdir}/python{py_version_short}/site-packages", - "include": "{installed_base}/include/python{py_version_short}{abiflags}", - "platinclude": "{installed_platbase}/include/python{py_version_short}{abiflags}", - "scripts": "{base}/bin", - "data": "{base}", - }, - "posix_home": { - "stdlib": "{installed_base}/lib/python", - "platstdlib": "{base}/lib/python", - "purelib": "{base}/lib/python", - "platlib": "{base}/lib/python", - "include": "{installed_base}/include/python", - "platinclude": "{installed_base}/include/python", - "scripts": "{base}/bin", - "data": "{base}", - }, - "nt": { - "stdlib": "{installed_base}/Lib", - "platstdlib": "{base}/Lib", - "purelib": "{base}/Lib/site-packages", - "platlib": "{base}/Lib/site-packages", - "include": "{installed_base}/Include", - "platinclude": "{installed_base}/Include", - "scripts": "{base}/Scripts", - "data": "{base}", - }, - "deb_system": { - "stdlib": "{installed_base}/{platlibdir}/python{py_version_short}", - "platstdlib": "{platbase}/{platlibdir}/python{py_version_short}", - "purelib": "{base}/lib/python3/dist-packages", - "platlib": "{platbase}/{platlibdir}/python3/dist-packages", - "include": "{installed_base}/include/python{py_version_short}{abiflags}", - "platinclude": "{installed_platbase}/include/python{py_version_short}{abiflags}", - "scripts": "{base}/bin", - "data": "{base}", - }, - "posix_local": { - "stdlib": "{installed_base}/{platlibdir}/python{py_version_short}", - "platstdlib": "{platbase}/{platlibdir}/python{py_version_short}", - "purelib": "{base}/local/lib/python{py_version_short}/dist-packages", - "platlib": "{platbase}/local/lib/python{py_version_short}/dist-packages", - "include": "{installed_base}/local/include/python{py_version_short}{abiflags}", - "platinclude": "{installed_platbase}/local/include/python{py_version_short}{abiflags}", - "scripts": "{base}/local/bin", - "data": "{base}", - }, - } - # reset the default in case we're on a system which doesn't have this problem - sysconfig_get_path = functools.partial(sysconfig.get_path, scheme="posix_local") - - # make it look like python3-distutils is not available - mocker.patch.dict(sys.modules, {"distutils.command": None}) - mocker.patch("sysconfig._INSTALL_SCHEMES", sysconfig_install_schemes) - mocker.patch("sysconfig.get_path", sysconfig_get_path) - mocker.patch("sysconfig.get_default_scheme", return_value="posix_local") - - pyinfo = PythonInfo() - pyver = f"{pyinfo.version_info.major}.{pyinfo.version_info.minor}" - assert pyinfo.install_path("scripts") == "bin" - assert pyinfo.install_path("purelib").replace(os.sep, "/") == f"lib/python{pyver}/site-packages" diff --git a/tests/unit/discovery/py_info/test_py_info_exe_based_of.py b/tests/unit/discovery/py_info/test_py_info_exe_based_of.py deleted file mode 100644 index 90894a59c..000000000 --- a/tests/unit/discovery/py_info/test_py_info_exe_based_of.py +++ /dev/null @@ -1,56 +0,0 @@ -from __future__ import annotations - -import logging -import os -from pathlib import Path - -import pytest - -from virtualenv.discovery.py_info import EXTENSIONS, PythonInfo -from virtualenv.info import IS_WIN, fs_is_case_sensitive, fs_supports_symlink - -CURRENT = PythonInfo.current() - - -def test_discover_empty_folder(tmp_path, session_app_data): - with pytest.raises(RuntimeError): - CURRENT.discover_exe(session_app_data, prefix=str(tmp_path)) - - -BASE = (CURRENT.install_path("scripts"), ".") - - -@pytest.mark.skipif(not fs_supports_symlink(), reason="symlink is not supported") -@pytest.mark.parametrize("suffix", sorted({".exe", ".cmd", ""} & set(EXTENSIONS) if IS_WIN else [""])) -@pytest.mark.parametrize("into", BASE) -@pytest.mark.parametrize("arch", [CURRENT.architecture, ""]) -@pytest.mark.parametrize("version", [".".join(str(i) for i in CURRENT.version_info[0:i]) for i in range(3, 0, -1)]) -@pytest.mark.parametrize("impl", [CURRENT.implementation, "python"]) -def test_discover_ok(tmp_path, suffix, impl, version, arch, into, caplog, session_app_data): # noqa: PLR0913 - caplog.set_level(logging.DEBUG) - folder = tmp_path / into - folder.mkdir(parents=True, exist_ok=True) - name = f"{impl}{version}{'t' if CURRENT.free_threaded else ''}" - if arch: - name += f"-{arch}" - name += suffix - dest = folder / name - os.symlink(CURRENT.executable, str(dest)) - pyvenv = Path(CURRENT.executable).parents[1] / "pyvenv.cfg" - if pyvenv.exists(): - (folder / pyvenv.name).write_text(pyvenv.read_text(encoding="utf-8"), encoding="utf-8") - inside_folder = str(tmp_path) - base = CURRENT.discover_exe(session_app_data, inside_folder) - found = base.executable - dest_str = str(dest) - if not fs_is_case_sensitive(): - found = found.lower() - dest_str = dest_str.lower() - assert found == dest_str - assert len(caplog.messages) >= 1, caplog.text - assert "get interpreter info via cmd: " in caplog.text - - dest.rename(dest.parent / (dest.name + "-1")) - CURRENT._cache_exe_discovery.clear() # noqa: SLF001 - with pytest.raises(RuntimeError): - CURRENT.discover_exe(session_app_data, inside_folder) diff --git a/tests/unit/discovery/test_discovery.py b/tests/unit/discovery/test_discovery.py index e6c78e2e3..a5aba7e8f 100644 --- a/tests/unit/discovery/test_discovery.py +++ b/tests/unit/discovery/test_discovery.py @@ -2,81 +2,20 @@ import logging import os -import subprocess import sys from argparse import Namespace from pathlib import Path -from unittest.mock import patch -from uuid import uuid4 import pytest +from python_discovery import PythonInfo -from virtualenv.discovery.builtin import Builtin, LazyPathDump, get_interpreter, get_paths -from virtualenv.discovery.py_info import PythonInfo -from virtualenv.info import IS_WIN, fs_supports_symlink +from virtualenv.discovery.builtin import Builtin, get_interpreter +from virtualenv.info import IS_WIN +from virtualenv.run import cli_run, session_via_cli -@pytest.mark.skipif(not fs_supports_symlink(), reason="symlink not supported") -@pytest.mark.parametrize("case", ["mixed", "lower", "upper"]) -@pytest.mark.parametrize("specificity", ["more", "less", "none"]) -def test_discovery_via_path(monkeypatch, case, specificity, tmp_path, caplog, session_app_data): # noqa: PLR0913 - caplog.set_level(logging.DEBUG) - current = PythonInfo.current_system(session_app_data) - name = "somethingVeryCryptic" - threaded = "t" if current.free_threaded else "" - if case == "lower": - name = name.lower() - elif case == "upper": - name = name.upper() - if specificity == "more": - # e.g. spec: python3, exe: /bin/python3.12 - core_ver = current.version_info.major - exe_ver = ".".join(str(i) for i in current.version_info[0:2]) + threaded - elif specificity == "less": - # e.g. spec: python3.12.1, exe: /bin/python3 - core_ver = ".".join(str(i) for i in current.version_info[0:3]) - exe_ver = current.version_info.major - elif specificity == "none": - # e.g. spec: python3.12.1, exe: /bin/python - core_ver = ".".join(str(i) for i in current.version_info[0:3]) - exe_ver = "" - core = "" if specificity == "none" else f"{name}{core_ver}{threaded}" - exe_name = f"{name}{exe_ver}{'.exe' if sys.platform == 'win32' else ''}" - target = tmp_path / current.install_path("scripts") - target.mkdir(parents=True) - executable = target / exe_name - os.symlink(sys.executable, str(executable)) - pyvenv_cfg = Path(sys.executable).parents[1] / "pyvenv.cfg" - if pyvenv_cfg.exists(): - (target / pyvenv_cfg.name).write_bytes(pyvenv_cfg.read_bytes()) - new_path = os.pathsep.join([str(target), *os.environ.get("PATH", "").split(os.pathsep)]) - monkeypatch.setenv("PATH", new_path) - interpreter = get_interpreter(core, []) - - assert interpreter is not None - - -def test_discovery_via_path_not_found(tmp_path, monkeypatch): - monkeypatch.setenv("PATH", str(tmp_path)) - interpreter = get_interpreter(uuid4().hex, []) - assert interpreter is None - - -def test_discovery_via_path_in_nonbrowseable_directory(tmp_path, monkeypatch): - bad_perm = tmp_path / "bad_perm" - bad_perm.mkdir(mode=0o000) - # path entry is unreadable - monkeypatch.setenv("PATH", str(bad_perm)) - interpreter = get_interpreter(uuid4().hex, []) - assert interpreter is None - # path entry parent is unreadable - monkeypatch.setenv("PATH", str(bad_perm / "bin")) - interpreter = get_interpreter(uuid4().hex, []) - assert interpreter is None - - -def test_relative_path(session_app_data, monkeypatch): - sys_executable = Path(PythonInfo.current_system(app_data=session_app_data).system_executable) +def test_relative_path(session_app_data, monkeypatch) -> None: + sys_executable = Path(PythonInfo.current_system(session_app_data).system_executable) cwd = sys_executable.parents[1] monkeypatch.chdir(str(cwd)) relative = str(sys_executable.relative_to(cwd)) @@ -84,69 +23,7 @@ def test_relative_path(session_app_data, monkeypatch): assert result is not None -def test_uv_python(monkeypatch, tmp_path_factory, mocker): - monkeypatch.delenv("UV_PYTHON_INSTALL_DIR", raising=False) - monkeypatch.delenv("XDG_DATA_HOME", raising=False) - monkeypatch.setenv("PATH", "") - mocker.patch.object(PythonInfo, "satisfies", return_value=False) - - # UV_PYTHON_INSTALL_DIR - uv_python_install_dir = tmp_path_factory.mktemp("uv_python_install_dir") - with patch("virtualenv.discovery.builtin.PathPythonInfo.from_exe") as mock_from_exe, monkeypatch.context() as m: - m.setenv("UV_PYTHON_INSTALL_DIR", str(uv_python_install_dir)) - - get_interpreter("python", []) - mock_from_exe.assert_not_called() - - bin_path = uv_python_install_dir.joinpath("some-py-impl", "bin") - bin_path.mkdir(parents=True) - bin_path.joinpath("python").touch() - get_interpreter("python", []) - mock_from_exe.assert_called_once() - assert mock_from_exe.call_args[0][0] == str(bin_path / "python") - - # PATH takes precedence - mock_from_exe.reset_mock() - python_exe = "python.exe" if IS_WIN else "python" - dir_in_path = tmp_path_factory.mktemp("path_bin_dir") - dir_in_path.joinpath(python_exe).touch() - m.setenv("PATH", str(dir_in_path)) - get_interpreter("python", []) - mock_from_exe.assert_called_once() - assert mock_from_exe.call_args[0][0] == str(dir_in_path / python_exe) - - # XDG_DATA_HOME - xdg_data_home = tmp_path_factory.mktemp("xdg_data_home") - with patch("virtualenv.discovery.builtin.PathPythonInfo.from_exe") as mock_from_exe, monkeypatch.context() as m: - m.setenv("XDG_DATA_HOME", str(xdg_data_home)) - - get_interpreter("python", []) - mock_from_exe.assert_not_called() - - bin_path = xdg_data_home.joinpath("uv", "python", "some-py-impl", "bin") - bin_path.mkdir(parents=True) - bin_path.joinpath("python").touch() - get_interpreter("python", []) - mock_from_exe.assert_called_once() - assert mock_from_exe.call_args[0][0] == str(bin_path / "python") - - # User data path - user_data_path = tmp_path_factory.mktemp("user_data_path") - with patch("virtualenv.discovery.builtin.PathPythonInfo.from_exe") as mock_from_exe, monkeypatch.context() as m: - m.setattr("virtualenv.discovery.builtin.user_data_path", lambda x: user_data_path / x) - - get_interpreter("python", []) - mock_from_exe.assert_not_called() - - bin_path = user_data_path.joinpath("uv", "python", "some-py-impl", "bin") - bin_path.mkdir(parents=True) - bin_path.joinpath("python").touch() - get_interpreter("python", []) - mock_from_exe.assert_called_once() - assert mock_from_exe.call_args[0][0] == str(bin_path / "python") - - -def test_discovery_fallback_fail(session_app_data, caplog): +def test_discovery_fallback_fail(session_app_data, caplog) -> None: caplog.set_level(logging.DEBUG) builtin = Builtin( Namespace(app_data=session_app_data, try_first_with=[], python=["magic-one", "magic-two"], env=os.environ), @@ -158,7 +35,7 @@ def test_discovery_fallback_fail(session_app_data, caplog): assert "accepted" not in caplog.text -def test_discovery_fallback_ok(session_app_data, caplog): +def test_discovery_fallback_ok(session_app_data, caplog) -> None: caplog.set_level(logging.DEBUG) builtin = Builtin( Namespace(app_data=session_app_data, try_first_with=[], python=["magic-one", sys.executable], env=os.environ), @@ -175,57 +52,45 @@ def test_discovery_fallback_ok(session_app_data, caplog): def mock_get_interpreter(mocker): return mocker.patch( "virtualenv.discovery.builtin.get_interpreter", - lambda key, *args, **kwargs: getattr(mocker.sentinel, key), # noqa: ARG005 + lambda key, *_args, **_kwargs: getattr(mocker.sentinel, key), ) @pytest.mark.usefixtures("mock_get_interpreter") -def test_returns_first_python_specified_when_only_env_var_one_is_specified(mocker, monkeypatch, session_app_data): - monkeypatch.setenv("VIRTUALENV_PYTHON", "python_from_env_var") - builtin = Builtin( - Namespace(app_data=session_app_data, try_first_with=[], python=["python_from_env_var"], env=os.environ), - ) - - result = builtin.run() - - assert result == mocker.sentinel.python_from_env_var - - -@pytest.mark.usefixtures("mock_get_interpreter") -def test_returns_second_python_specified_when_more_than_one_is_specified_and_env_var_is_specified( - mocker, monkeypatch, session_app_data -): - monkeypatch.setenv("VIRTUALENV_PYTHON", "python_from_env_var") - builtin = Builtin( - Namespace( - app_data=session_app_data, - try_first_with=[], - python=["python_from_env_var", "python_from_cli"], - env=os.environ, +@pytest.mark.parametrize( + ("env_var", "python", "expected"), + [ + pytest.param("python_from_env_var", ["python_from_env_var"], "python_from_env_var", id="env-var-only"), + pytest.param( + "python_from_env_var", ["python_from_env_var", "python_from_cli"], "python_from_cli", id="env-var-and-cli" ), - ) - - result = builtin.run() - - assert result == mocker.sentinel.python_from_cli - - -def test_discovery_absolute_path_with_try_first(tmp_path, session_app_data): + pytest.param(None, ["python_from_cli"], "python_from_cli", id="cli-only"), + ], +) +def test_returns_python_specified( # ruff:ignore[too-many-arguments] + mocker, monkeypatch, session_app_data, env_var: str | None, python: list[str], expected: str +) -> None: + if env_var is None: + monkeypatch.delenv("VIRTUALENV_PYTHON", raising=False) + else: + monkeypatch.setenv("VIRTUALENV_PYTHON", env_var) + builtin = Builtin(Namespace(app_data=session_app_data, try_first_with=[], python=python, env=os.environ)) + + assert builtin.run() == getattr(mocker.sentinel, expected) + + +def test_discovery_absolute_path_with_try_first(tmp_path, session_app_data) -> None: good_env = tmp_path / "good" bad_env = tmp_path / "bad" - # Create two real virtual environments - subprocess.check_call([sys.executable, "-m", "virtualenv", str(good_env)]) - subprocess.check_call([sys.executable, "-m", "virtualenv", str(bad_env)]) + cli_run(["--no-seed", str(good_env)]) + cli_run(["--no-seed", str(bad_env)]) - # On Windows, the executable is in Scripts/python.exe scripts_dir = "Scripts" if IS_WIN else "bin" exe_name = "python.exe" if IS_WIN else "python" good_exe = good_env / scripts_dir / exe_name bad_exe = bad_env / scripts_dir / exe_name - # The spec is an absolute path, this should be a hard requirement. - # The --try-first-with option should be rejected as it does not match the spec. interpreter = get_interpreter( str(good_exe), try_first_with=[str(bad_exe)], @@ -236,96 +101,38 @@ def test_discovery_absolute_path_with_try_first(tmp_path, session_app_data): assert Path(interpreter.executable) == good_exe -def test_discovery_via_path_with_file(tmp_path, monkeypatch): - a_file = tmp_path / "a_file" - a_file.touch() - monkeypatch.setenv("PATH", str(a_file)) - interpreter = get_interpreter(uuid4().hex, []) - assert interpreter is None - - -def test_absolute_path_does_not_exist(tmp_path): - """ - Test that virtualenv does not fail when an absolute path that does not exist is provided. - """ - # Create a command that uses an absolute path that does not exist - # and a valid python executable. - command = [ - sys.executable, - "-m", - "virtualenv", - "-p", - "/this/path/does/not/exist", - "-p", - sys.executable, - str(tmp_path / "dest"), - ] - - # Run the command - process = subprocess.run( - command, - capture_output=True, - text=True, - check=False, - encoding="utf-8", - ) - - # Check that the command was successful - assert process.returncode == 0, process.stderr - - -def test_absolute_path_does_not_exist_fails(tmp_path): - """ - Test that virtualenv fails when a single absolute path that does not exist is provided. - """ - # Create a command that uses an absolute path that does not exist - command = [ - sys.executable, - "-m", - "virtualenv", - "-p", - "/this/path/does/not/exist", - str(tmp_path / "dest"), - ] - - # Run the command - process = subprocess.run( - command, - capture_output=True, - text=True, - check=False, - encoding="utf-8", - ) - - # Check that the command failed - assert process.returncode != 0, process.stderr - +def test_absolute_path_does_not_exist(tmp_path) -> None: + """Test that virtualenv does not fail when an absolute path that does not exist is provided.""" + session = session_via_cli(["-p", "/this/path/does/not/exist", "-p", sys.executable, str(tmp_path / "dest")]) -def test_get_paths_no_path_env(monkeypatch): - monkeypatch.delenv("PATH", raising=False) - paths = list(get_paths({})) - assert paths + assert session.interpreter.executable == sys.executable -def test_lazy_path_dump_debug(monkeypatch, tmp_path): - monkeypatch.setenv("_VIRTUALENV_DEBUG", "1") - a_dir = tmp_path - executable_file = "a_file.exe" if IS_WIN else "a_file" - (a_dir / executable_file).touch(mode=0o755) - (a_dir / "b_file").touch(mode=0o644) - dumper = LazyPathDump(0, a_dir, os.environ) - output = repr(dumper) - assert executable_file in output - assert "b_file" not in output +def test_absolute_path_does_not_exist_fails(tmp_path) -> None: + """Test that virtualenv fails when a single absolute path that does not exist is provided.""" + with pytest.raises(RuntimeError, match="failed to find interpreter"): + session_via_cli(["-p", "/this/path/does/not/exist", str(tmp_path / "dest")]) -@pytest.mark.usefixtures("mock_get_interpreter") -def test_returns_first_python_specified_when_no_env_var_is_specified(mocker, monkeypatch, session_app_data): - monkeypatch.delenv("VIRTUALENV_PYTHON", raising=False) - builtin = Builtin( - Namespace(app_data=session_app_data, try_first_with=[], python=["python_from_cli"], env=os.environ), - ) +def test_discovery_via_version_specifier(session_app_data) -> None: + """Test that version specifiers like >=3.11 work correctly through the virtualenv wrapper.""" + current = PythonInfo.current_system(session_app_data) + major, minor = current.version_info.major, current.version_info.minor - result = builtin.run() + spec = f">={major}.{minor}" + interpreter = get_interpreter(spec, [], session_app_data) + assert interpreter is not None + assert interpreter.version_info.major == major + assert interpreter.version_info.minor >= minor - assert result == mocker.sentinel.python_from_cli + spec = f">={major}.{minor},<{major}.{minor + 10}" + interpreter = get_interpreter(spec, [], session_app_data) + assert interpreter is not None + assert interpreter.version_info.major == major + assert minor <= interpreter.version_info.minor < minor + 10 + + spec = f"cpython>={major}.{minor}" + interpreter = get_interpreter(spec, [], session_app_data) + if current.implementation == "CPython": + assert interpreter is not None + assert interpreter.implementation == "CPython" diff --git a/tests/unit/discovery/test_py_spec.py b/tests/unit/discovery/test_py_spec.py deleted file mode 100644 index 0841019ec..000000000 --- a/tests/unit/discovery/test_py_spec.py +++ /dev/null @@ -1,134 +0,0 @@ -from __future__ import annotations - -import sys -from copy import copy - -import pytest - -from virtualenv.discovery.py_spec import PythonSpec - - -def test_bad_py_spec(): - text = "python2.3.4.5" - spec = PythonSpec.from_string_spec(text) - assert text in repr(spec) - assert spec.str_spec == text - assert spec.path == text - content = vars(spec) - del content["str_spec"] - del content["path"] - assert all(v is None for v in content.values()) - - -def test_py_spec_first_digit_only_major(): - spec = PythonSpec.from_string_spec("278") - assert spec.major == 2 - assert spec.minor == 78 - - -def test_spec_satisfies_path_ok(): - spec = PythonSpec.from_string_spec(sys.executable) - assert spec.satisfies(spec) is True - - -def test_spec_satisfies_path_nok(tmp_path): - spec = PythonSpec.from_string_spec(sys.executable) - of = PythonSpec.from_string_spec(str(tmp_path)) - assert spec.satisfies(of) is False - - -def test_spec_satisfies_arch(): - spec_1 = PythonSpec.from_string_spec("python-32") - spec_2 = PythonSpec.from_string_spec("python-64") - - assert spec_1.satisfies(spec_1) is True - assert spec_2.satisfies(spec_1) is False - - -def test_spec_satisfies_free_threaded(): - spec_1 = PythonSpec.from_string_spec("python3.13t") - spec_2 = PythonSpec.from_string_spec("python3.13") - - assert spec_1.satisfies(spec_1) is True - assert spec_1.free_threaded is True - assert spec_2.satisfies(spec_1) is False - assert spec_2.free_threaded is False - - -@pytest.mark.parametrize( - ("req", "spec"), - [("py", "python"), ("jython", "jython"), ("CPython", "cpython")], -) -def test_spec_satisfies_implementation_ok(req, spec): - spec_1 = PythonSpec.from_string_spec(req) - spec_2 = PythonSpec.from_string_spec(spec) - assert spec_1.satisfies(spec_1) is True - assert spec_2.satisfies(spec_1) is True - - -def test_spec_satisfies_implementation_nok(): - spec_1 = PythonSpec.from_string_spec("cpython") - spec_2 = PythonSpec.from_string_spec("jython") - assert spec_2.satisfies(spec_1) is False - assert spec_1.satisfies(spec_2) is False - - -def _version_satisfies_pairs(): - target = set() - version = tuple(str(i) for i in sys.version_info[0:3]) - for threading in (False, True): - for i in range(len(version) + 1): - req = ".".join(version[0:i]) - for j in range(i + 1): - sat = ".".join(version[0:j]) - # can be satisfied in both directions - if sat: - target.add((req, sat)) - # else: no version => no free-threading info - target.add((sat, req)) - if not threading or not sat or not req: - # free-threading info requires a version - continue - target.add((f"{req}t", f"{sat}t")) - target.add((f"{sat}t", f"{req}t")) - - return sorted(target) - - -@pytest.mark.parametrize(("req", "spec"), _version_satisfies_pairs()) -def test_version_satisfies_ok(req, spec): - req_spec = PythonSpec.from_string_spec(f"python{req}") - sat_spec = PythonSpec.from_string_spec(f"python{spec}") - assert sat_spec.satisfies(req_spec) is True - - -def _version_not_satisfies_pairs(): - target = set() - version = tuple(str(i) for i in sys.version_info[0:3]) - for major in range(len(version)): - req = ".".join(version[0 : major + 1]) - for minor in range(major + 1): - sat_ver = list(sys.version_info[0 : minor + 1]) - for patch in range(minor + 1): - for o in [1, -1]: - temp = copy(sat_ver) - temp[patch] += o - if temp[patch] < 0: - continue - sat = ".".join(str(i) for i in temp) - target.add((req, sat)) - return sorted(target) - - -@pytest.mark.parametrize(("req", "spec"), _version_not_satisfies_pairs()) -def test_version_satisfies_nok(req, spec): - req_spec = PythonSpec.from_string_spec(f"python{req}") - sat_spec = PythonSpec.from_string_spec(f"python{spec}") - assert sat_spec.satisfies(req_spec) is False - - -def test_relative_spec(tmp_path, monkeypatch): - monkeypatch.chdir(tmp_path) - a_relative_path = str((tmp_path / "a" / "b").relative_to(tmp_path)) - spec = PythonSpec.from_string_spec(a_relative_path) - assert spec.path == a_relative_path diff --git a/tests/unit/discovery/windows/conftest.py b/tests/unit/discovery/windows/conftest.py deleted file mode 100644 index f75278e06..000000000 --- a/tests/unit/discovery/windows/conftest.py +++ /dev/null @@ -1,105 +0,0 @@ -from __future__ import annotations - -from contextlib import contextmanager -from pathlib import Path - -import pytest - - -@pytest.fixture -def _mock_registry(mocker): # noqa: C901 - from virtualenv.discovery.windows.pep514 import winreg # noqa: PLC0415 - - loc, glob = {}, {} - mock_value_str = (Path(__file__).parent / "winreg-mock-values.py").read_text(encoding="utf-8") - exec(mock_value_str, glob, loc) # noqa: S102 - enum_collect = loc["enum_collect"] - value_collect = loc["value_collect"] - key_open = loc["key_open"] - hive_open = loc["hive_open"] - - def _enum_key(key, at): - key_id = key.value if isinstance(key, Key) else key - result = enum_collect[key_id][at] - if isinstance(result, OSError): - raise result - return result - - mocker.patch.object(winreg, "EnumKey", side_effect=_enum_key) - - def _query_value_ex(key, value_name): - key_id = key.value if isinstance(key, Key) else key - result = value_collect[key_id][value_name] - if isinstance(result, OSError): - raise result - return result - - mocker.patch.object(winreg, "QueryValueEx", side_effect=_query_value_ex) - - class Key: - def __init__(self, value) -> None: - self.value = value - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - return None - - @contextmanager - def _open_key_ex(*args): - if len(args) == 2: - key, value = args - key_id = key.value if isinstance(key, Key) else key - result = Key(key_open[key_id][value]) # this needs to be something that can be with-ed, so let's wrap it - elif len(args) == 4: - result = hive_open[args] - else: - raise RuntimeError - value = result.value if isinstance(result, Key) else result - if isinstance(value, OSError): - raise value - yield result - - mocker.patch.object(winreg, "OpenKeyEx", side_effect=_open_key_ex) - mocker.patch("os.path.exists", return_value=True) - - -def _mock_pyinfo(major, minor, arch, exe, threaded=False): - """Return PythonInfo objects with essential metadata set for the given args""" - from virtualenv.discovery.py_info import PythonInfo, VersionInfo # noqa: PLC0415 - - info = PythonInfo() - info.base_prefix = str(Path(exe).parent) - info.executable = info.original_executable = info.system_executable = exe - info.implementation = "CPython" - info.architecture = arch - info.version_info = VersionInfo(major, minor, 0, "final", 0) - info.free_threaded = threaded - return info - - -@pytest.fixture -def _populate_pyinfo_cache(monkeypatch): - """Add metadata to virtualenv.discovery.cached_py_info._CACHE for all (mocked) registry entries""" - import virtualenv.discovery.cached_py_info # noqa: PLC0415 - - # Data matches _mock_registry fixture - python_core_path = "C:\\Users\\user\\AppData\\Local\\Programs\\Python" - interpreters = [ - ("ContinuumAnalytics", 3, 10, 32, False, "C:\\Users\\user\\Miniconda3\\python.exe"), - ("ContinuumAnalytics", 3, 10, 64, False, "C:\\Users\\user\\Miniconda3-64\\python.exe"), - ("PythonCore", 3, 9, 64, False, f"{python_core_path}\\Python36\\python.exe"), - ("PythonCore", 3, 9, 64, False, f"{python_core_path}\\Python36\\python.exe"), - ("PythonCore", 3, 5, 64, False, f"{python_core_path}\\Python35\\python.exe"), - ("PythonCore", 3, 9, 64, False, f"{python_core_path}\\Python36\\python.exe"), - ("PythonCore", 3, 7, 32, False, f"{python_core_path}\\Python37-32\\python.exe"), - ("PythonCore", 3, 12, 64, False, f"{python_core_path}\\Python312\\python.exe"), - ("PythonCore", 3, 13, 64, True, f"{python_core_path}\\Python313\\python3.13t.exe"), - ("PythonCore", 2, 7, 64, False, "C:\\Python27\\python.exe"), - ("PythonCore", 3, 4, 64, False, "C:\\Python34\\python.exe"), - ("CompanyA", 3, 6, 64, False, "Z:\\CompanyA\\Python\\3.6\\python.exe"), - ] - for _, major, minor, arch, threaded, exe in interpreters: - info = _mock_pyinfo(major, minor, arch, exe, threaded) - monkeypatch.setitem(virtualenv.discovery.cached_py_info._CACHE, Path(info.executable), info) # noqa: SLF001 diff --git a/tests/unit/discovery/windows/test_windows.py b/tests/unit/discovery/windows/test_windows.py deleted file mode 100644 index 594a1302f..000000000 --- a/tests/unit/discovery/windows/test_windows.py +++ /dev/null @@ -1,40 +0,0 @@ -from __future__ import annotations - -import sys - -import pytest - -from virtualenv.discovery.py_spec import PythonSpec - - -@pytest.mark.skipif(sys.platform != "win32", reason="no Windows registry") -@pytest.mark.usefixtures("_mock_registry") -@pytest.mark.usefixtures("_populate_pyinfo_cache") -@pytest.mark.parametrize( - ("string_spec", "expected_exe"), - [ - # 64-bit over 32-bit - ("python3.10", "C:\\Users\\user\\Miniconda3-64\\python.exe"), - ("cpython3.10", "C:\\Users\\user\\Miniconda3-64\\python.exe"), - # 1 installation of 3.9 available - ("python3.12", "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python312\\python.exe"), - ("cpython3.12", "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python312\\python.exe"), - # resolves to highest available version - ("python", "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python313\\python3.13t.exe"), - ("cpython", "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python313\\python3.13t.exe"), - ("python3", "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python312\\python.exe"), - ("cpython3", "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python312\\python.exe"), - # Non-standard org name - ("python3.6", "Z:\\CompanyA\\Python\\3.6\\python.exe"), - ("cpython3.6", "Z:\\CompanyA\\Python\\3.6\\python.exe"), - # free-threaded - ("3t", "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python313\\python3.13t.exe"), - ("python3.13t", "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python313\\python3.13t.exe"), - ], -) -def test_propose_interpreters(string_spec, expected_exe): - from virtualenv.discovery.windows import propose_interpreters # noqa: PLC0415 - - spec = PythonSpec.from_string_spec(string_spec) - interpreter = next(propose_interpreters(spec=spec, cache_dir=None, env=None)) - assert interpreter.executable == expected_exe diff --git a/tests/unit/discovery/windows/test_windows_pep514.py b/tests/unit/discovery/windows/test_windows_pep514.py deleted file mode 100644 index 0498352aa..000000000 --- a/tests/unit/discovery/windows/test_windows_pep514.py +++ /dev/null @@ -1,123 +0,0 @@ -from __future__ import annotations - -import sys -import textwrap - -import pytest - - -@pytest.mark.skipif(sys.platform != "win32", reason="no Windows registry") -@pytest.mark.usefixtures("_mock_registry") -def test_pep514(): - from virtualenv.discovery.windows.pep514 import discover_pythons # noqa: PLC0415 - - interpreters = list(discover_pythons()) - assert interpreters == [ - ("ContinuumAnalytics", 3, 10, 32, False, "C:\\Users\\user\\Miniconda3\\python.exe", None), - ("ContinuumAnalytics", 3, 10, 64, False, "C:\\Users\\user\\Miniconda3-64\\python.exe", None), - ( - "PythonCore", - 3, - 9, - 64, - False, - "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python39\\python.exe", - None, - ), - ( - "PythonCore", - 3, - 9, - 64, - False, - "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python39\\python.exe", - None, - ), - ( - "PythonCore", - 3, - 8, - 64, - False, - "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python38\\python.exe", - None, - ), - ( - "PythonCore", - 3, - 9, - 64, - False, - "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python39\\python.exe", - None, - ), - ( - "PythonCore", - 3, - 10, - 32, - False, - "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python310-32\\python.exe", - None, - ), - ( - "PythonCore", - 3, - 12, - 64, - False, - "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python312\\python.exe", - None, - ), - ( - "PythonCore", - 3, - 13, - 64, - True, - "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python313\\python3.13t.exe", - None, - ), - ("CompanyA", 3, 6, 64, False, "Z:\\CompanyA\\Python\\3.6\\python.exe", None), - ("PythonCore", 2, 7, 64, False, "C:\\Python27\\python.exe", None), - ("PythonCore", 3, 7, 64, False, "C:\\Python37\\python.exe", None), - ] - - -@pytest.mark.skipif(sys.platform != "win32", reason="no Windows registry") -@pytest.mark.usefixtures("_mock_registry") -def test_pep514_run(capsys, caplog): - from virtualenv.discovery.windows import pep514 # noqa: PLC0415 - - pep514._run() # noqa: SLF001 - out, err = capsys.readouterr() - expected = textwrap.dedent( - r""" - ('CompanyA', 3, 6, 64, False, 'Z:\\CompanyA\\Python\\3.6\\python.exe', None) - ('ContinuumAnalytics', 3, 10, 32, False, 'C:\\Users\\user\\Miniconda3\\python.exe', None) - ('ContinuumAnalytics', 3, 10, 64, False, 'C:\\Users\\user\\Miniconda3-64\\python.exe', None) - ('PythonCore', 2, 7, 64, False, 'C:\\Python27\\python.exe', None) - ('PythonCore', 3, 10, 32, False, 'C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python310-32\\python.exe', None) - ('PythonCore', 3, 12, 64, False, 'C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python312\\python.exe', None) - ('PythonCore', 3, 13, 64, True, 'C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python313\\python3.13t.exe', None) - ('PythonCore', 3, 7, 64, False, 'C:\\Python37\\python.exe', None) - ('PythonCore', 3, 8, 64, False, 'C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python38\\python.exe', None) - ('PythonCore', 3, 9, 64, False, 'C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python39\\python.exe', None) - ('PythonCore', 3, 9, 64, False, 'C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python39\\python.exe', None) - ('PythonCore', 3, 9, 64, False, 'C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python39\\python.exe', None) - """, # noqa: E501 - ).strip() - assert out.strip() == expected - assert not err - prefix = "PEP-514 violation in Windows Registry at " - expected_logs = [ - f"{prefix}HKEY_CURRENT_USER/PythonCore/3.1/SysArchitecture error: invalid format magic", - f"{prefix}HKEY_CURRENT_USER/PythonCore/3.2/SysArchitecture error: arch is not string: 100", - f"{prefix}HKEY_CURRENT_USER/PythonCore/3.3 error: no ExecutablePath or default for it", - f"{prefix}HKEY_CURRENT_USER/PythonCore/3.3 error: could not load exe with value None", - f"{prefix}HKEY_CURRENT_USER/PythonCore/3.11/InstallPath error: missing", - f"{prefix}HKEY_CURRENT_USER/PythonCore/3.12/SysVersion error: invalid format magic", - f"{prefix}HKEY_CURRENT_USER/PythonCore/3.X/SysVersion error: version is not string: 2778", - f"{prefix}HKEY_CURRENT_USER/PythonCore/3.X error: invalid format 3.X", - ] - assert caplog.messages == expected_logs diff --git a/tests/unit/discovery/windows/winreg-mock-values.py b/tests/unit/discovery/windows/winreg-mock-values.py deleted file mode 100644 index fa2619ba7..000000000 --- a/tests/unit/discovery/windows/winreg-mock-values.py +++ /dev/null @@ -1,172 +0,0 @@ -from __future__ import annotations - -import winreg - -hive_open = { - (winreg.HKEY_CURRENT_USER, "Software\\Python", 0, winreg.KEY_READ): 78701856, - (winreg.HKEY_LOCAL_MACHINE, "Software\\Python", 0, winreg.KEY_READ | winreg.KEY_WOW64_64KEY): 78701840, - (winreg.HKEY_LOCAL_MACHINE, "Software\\Python", 0, winreg.KEY_READ | winreg.KEY_WOW64_32KEY): OSError( - 2, - "The system cannot find the file specified", - ), -} -key_open = { - 78701152: { - "Anaconda310-32\\InstallPath": 78703200, - "Anaconda310-32": 78703568, - "Anaconda310-64\\InstallPath": 78703520, - "Anaconda310-64": 78702368, - }, - 78701856: {"ContinuumAnalytics": 78701152, "PythonCore": 78702656, "CompanyA": 88800000}, - 78702656: { - "3.1\\InstallPath": 78701824, - "3.1": 78700704, - "3.2\\InstallPath": 78704048, - "3.2": 78704368, - "3.3\\InstallPath": 78701936, - "3.3": 78703024, - "3.8\\InstallPath": 78703792, - "3.8": 78701792, - "3.9\\InstallPath": 78701888, - "3.9": 78703424, - "3.10-32\\InstallPath": 78703600, - "3.10-32": 78704512, - "3.11\\InstallPath": OSError(2, "The system cannot find the file specified"), - "3.11": 78700656, - "3.12\\InstallPath": 78703632, - "3.12": 78702608, - "3.13t\\InstallPath": 78703633, - "3.13t": 78702609, - "3.X": 78703088, - }, - 78702960: {"2.7\\InstallPath": 78700912, "2.7": 78703136, "3.7\\InstallPath": 78703648, "3.7": 78704032}, - 78701840: {"PythonCore": 78702960}, - 88800000: { - "3.6\\InstallPath": 88810000, - "3.6": 88820000, - }, -} -value_collect = { - 78703568: {"SysVersion": ("3.10", 1), "SysArchitecture": ("32bit", 1), "DisplayName": ("Python 3.10 (32-bit)", 1)}, - 78703200: { - "ExecutablePath": ("C:\\Users\\user\\Miniconda3\\python.exe", 1), - "ExecutableArguments": OSError(2, "The system cannot find the file specified"), - }, - 78702368: {"SysVersion": ("3.10", 1), "SysArchitecture": ("64bit", 1), "DisplayName": ("Python 3.10 (64-bit)", 1)}, - 78703520: { - "ExecutablePath": ("C:\\Users\\user\\Miniconda3-64\\python.exe", 1), - "ExecutableArguments": OSError(2, "The system cannot find the file specified"), - }, - 78700704: {"SysVersion": ("3.9", 1), "SysArchitecture": ("magic", 1), "DisplayName": ("Python 3.9 (wizardry)", 1)}, - 78701824: { - "ExecutablePath": ("C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python39\\python.exe", 1), - "ExecutableArguments": OSError(2, "The system cannot find the file specified"), - }, - 78704368: {"SysVersion": ("3.9", 1), "SysArchitecture": (100, 4), "DisplayName": ("Python 3.9 (64-bit)", 1)}, - 78704048: { - "ExecutablePath": ("C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python39\\python.exe", 1), - "ExecutableArguments": OSError(2, "The system cannot find the file specified"), - }, - 78703024: {"SysVersion": ("3.9", 1), "SysArchitecture": ("64bit", 1), "DisplayName": ("Python 3.9 (64-bit)", 1)}, - 78701936: { - "ExecutablePath": OSError(2, "The system cannot find the file specified"), - None: OSError(2, "The system cannot find the file specified"), - }, - 78701792: { - "SysVersion": OSError(2, "The system cannot find the file specified"), - "SysArchitecture": OSError(2, "The system cannot find the file specified"), - "DisplayName": OSError(2, "The system cannot find the file specified"), - }, - 78703792: { - "ExecutablePath": ("C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python38\\python.exe", 1), - "ExecutableArguments": OSError(2, "The system cannot find the file specified"), - }, - 78703424: {"SysVersion": ("3.9", 1), "SysArchitecture": ("64bit", 1), "DisplayName": ("Python 3.9 (64-bit)", 1)}, - 78701888: { - "ExecutablePath": ("C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python39\\python.exe", 1), - "ExecutableArguments": OSError(2, "The system cannot find the file specified"), - }, - 78704512: {"SysVersion": ("3.10", 1), "SysArchitecture": ("32bit", 1), "DisplayName": ("Python 3.10 (32-bit)", 1)}, - 78703600: { - "ExecutablePath": ("C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python310-32\\python.exe", 1), - "ExecutableArguments": OSError(2, "The system cannot find the file specified"), - }, - 78700656: { - "SysVersion": OSError(2, "The system cannot find the file specified"), - "SysArchitecture": OSError(2, "The system cannot find the file specified"), - "DisplayName": OSError(2, "The system cannot find the file specified"), - }, - 78702608: { - "SysVersion": ("magic", 1), - "SysArchitecture": ("64bit", 1), - "DisplayName": ("Python 3.12 (wizard edition)", 1), - }, - 78703632: { - "ExecutablePath": ("C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python312\\python.exe", 1), - "ExecutableArguments": OSError(2, "The system cannot find the file specified"), - }, - 78702609: { - "SysVersion": ("3.13", 1), - "SysArchitecture": ("64bit", 1), - "DisplayName": ("Python 3.13 (64-bit, freethreaded)", 1), - }, - 78703633: { - "ExecutablePath": ("C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python313\\python3.13t.exe", 1), - "ExecutableArguments": OSError(2, "The system cannot find the file specified"), - }, - 78703088: {"SysVersion": (2778, 11)}, - 78703136: { - "SysVersion": OSError(2, "The system cannot find the file specified"), - "SysArchitecture": OSError(2, "The system cannot find the file specified"), - "DisplayName": OSError(2, "The system cannot find the file specified"), - }, - 78700912: { - "ExecutablePath": OSError(2, "The system cannot find the file specified"), - None: ("C:\\Python27\\", 1), - "ExecutableArguments": OSError(2, "The system cannot find the file specified"), - }, - 78704032: { - "SysVersion": OSError(2, "The system cannot find the file specified"), - "SysArchitecture": OSError(2, "The system cannot find the file specified"), - "DisplayName": OSError(2, "The system cannot find the file specified"), - }, - 78703648: { - "ExecutablePath": OSError(2, "The system cannot find the file specified"), - None: ("C:\\Python37\\", 1), - "ExecutableArguments": OSError(2, "The system cannot find the file specified"), - }, - 88810000: { - "ExecutablePath": ("Z:\\CompanyA\\Python\\3.6\\python.exe", 1), - "ExecutableArguments": OSError(2, "The system cannot find the file specified"), - }, - 88820000: { - "SysVersion": ("3.6", 1), - "SysArchitecture": ("64bit", 1), - "DisplayName": OSError(2, "The system cannot find the file specified"), - }, -} -enum_collect = { - 78701856: [ - "ContinuumAnalytics", - "PythonCore", - "CompanyA", - OSError(22, "No more data is available", None, 259, None), - ], - 78701152: ["Anaconda310-32", "Anaconda310-64", OSError(22, "No more data is available", None, 259, None)], - 78702656: [ - "3.1", - "3.2", - "3.3", - "3.8", - "3.9", - "3.10-32", - "3.11", - "3.12", - "3.13t", - "3.X", - OSError(22, "No more data is available", None, 259, None), - ], - 78701840: ["PyLauncher", "PythonCore", OSError(22, "No more data is available", None, 259, None)], - 78702960: ["2.7", "3.7", OSError(22, "No more data is available", None, 259, None)], - 88800000: ["3.6", OSError(22, "No more data is available", None, 259, None)], -} diff --git a/tests/unit/seed/embed/test_base_embed.py b/tests/unit/seed/embed/test_base_embed.py index 4f4fadac9..0806ee5dc 100644 --- a/tests/unit/seed/embed/test_base_embed.py +++ b/tests/unit/seed/embed/test_base_embed.py @@ -1,65 +1,103 @@ from __future__ import annotations import sys -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, NamedTuple import pytest from virtualenv.run import session_via_cli +from virtualenv.seed.embed.pip_invoke import PipInvoke +from virtualenv.seed.embed.via_app_data.via_app_data import FromAppData +from virtualenv.seed.seeder import Seeder +from virtualenv.seed.wheels.embed import MIN, OLDEST_SUPPORTED if TYPE_CHECKING: + from collections.abc import Callable from pathlib import Path + from unittest.mock import MagicMock + + from pytest_mock import MockerFixture + + from virtualenv.seed.embed.base_embed import BaseEmbed @pytest.mark.parametrize( ("args", "download"), [([], False), (["--no-download"], False), (["--never-download"], False), (["--download"], True)], ) -def test_download_cli_flag(args, download, tmp_path): +def test_download_cli_flag(args, download, tmp_path) -> None: session = session_via_cli([*args, str(tmp_path)]) assert session.seeder.download is download -@pytest.mark.skipif(sys.version_info[:2] == (3, 8), reason="We still bundle wheel for Python 3.8") -@pytest.mark.parametrize("flag", ["--no-wheel", "--wheel=none", "--wheel=embed", "--wheel=bundle"]) -def test_wheel_cli_flags_do_nothing(tmp_path, flag): - session = session_via_cli([flag, str(tmp_path)]) - if sys.version_info[:2] >= (3, 12): - expected = {"pip": "bundle"} - else: - expected = {"pip": "bundle", "setuptools": "bundle"} - assert session.seeder.distribution_to_versions() == expected +@pytest.mark.parametrize( + ("flags", "warns"), + [ + pytest.param([], False, id="no-flag"), + pytest.param(["--no-wheel"], True, id="no-wheel"), + pytest.param(["--wheel=none"], True, id="wheel-none"), + pytest.param(["--wheel=embed"], True, id="wheel-embed"), + pytest.param(["--wheel=bundle"], True, id="wheel-bundle"), + ], +) +def test_wheel_cli_flags_do_nothing( + tmp_path: Path, capsys: pytest.CaptureFixture[str], flags: list[str], warns: bool +) -> None: + session = session_via_cli([*flags, str(tmp_path)]) + out, err = capsys.readouterr() + expected = {"pip": "bundle"} if sys.version_info[:2] >= (3, 12) else {"pip": "bundle", "setuptools": "bundle"} + assert ( + session.seeder.distribution_to_versions(), + "the --wheel and --no-wheel options do nothing" in out + err, + ) == (expected, warns) + + +BUNDLED_SEEDERS = [pytest.param(FromAppData, id="app-data"), pytest.param(PipInvoke, id="pip")] -@pytest.mark.skipif(sys.version_info[:2] == (3, 8), reason="We still bundle wheel for Python 3.8") -@pytest.mark.parametrize("flag", ["--no-wheel", "--wheel=none", "--wheel=embed", "--wheel=bundle"]) -def test_wheel_cli_flags_warn(tmp_path, flag, capsys): - session_via_cli([flag, str(tmp_path)]) - out, err = capsys.readouterr() - assert "The --no-wheel and --wheel options are deprecated." in out + err +@pytest.mark.parametrize("seeder", BUNDLED_SEEDERS) +def test_bundled_seeder_reports_reason_below_floor( + seeder: type[BaseEmbed], at_version: Callable[[int, int], MagicMock] +) -> None: + reason = seeder.cannot_seed(at_version(3, 8)) + assert reason is not None + assert "Python 3.8" in reason + assert f"Python {MIN}" in reason + assert "--no-seed" in reason -@pytest.mark.skipif(sys.version_info[:2] == (3, 8), reason="We still bundle wheel for Python 3.8") -def test_unused_wheel_cli_flags_dont_warn(tmp_path, capsys): - session_via_cli([str(tmp_path)]) - out, err = capsys.readouterr() - assert "The --no-wheel and --wheel options are deprecated." not in out + err +@pytest.mark.parametrize("seeder", BUNDLED_SEEDERS) +def test_bundled_seeder_has_no_reason_on_oldest_supported( + seeder: type[BaseEmbed], at_version: Callable[[int, int], MagicMock] +) -> None: + assert seeder.cannot_seed(at_version(*OLDEST_SUPPORTED)) is None -@pytest.mark.skipif(sys.version_info[:2] != (3, 8), reason="We only bundle wheel for Python 3.8") -@pytest.mark.parametrize("flag", ["--no-wheel", "--wheel=none", "--wheel=embed", "--wheel=bundle"]) -def test_wheel_cli_flags_dont_warn_on_38(tmp_path, flag, capsys): - session_via_cli([flag, str(tmp_path)]) - out, err = capsys.readouterr() - assert "The --no-wheel and --wheel options are deprecated." not in out + err - - -def test_embed_wheel_versions(tmp_path: Path) -> None: - session = session_via_cli([str(tmp_path)]) - if sys.version_info[:2] >= (3, 12): - expected = {"pip": "bundle"} - elif sys.version_info[:2] >= (3, 9): - expected = {"pip": "bundle", "setuptools": "bundle"} - else: - expected = {"pip": "bundle", "setuptools": "bundle", "wheel": "bundle"} - assert session.seeder.distribution_to_versions() == expected +def test_base_seeder_never_blocks(at_version: Callable[[int, int], MagicMock]) -> None: + assert Seeder.cannot_seed(at_version(3, 8)) is None + + +def test_selection_surfaces_the_seeder_reason(tmp_path: Path, mocker: MockerFixture) -> None: + mocker.patch.object(FromAppData, "cannot_seed", return_value="seeder said no for a specific reason") + with pytest.raises(RuntimeError, match="seeder said no for a specific reason"): + session_via_cli([str(tmp_path)]) + + +def test_no_seed_bypasses_capability_check(tmp_path: Path, mocker: MockerFixture) -> None: + mocker.patch.object(FromAppData, "cannot_seed", return_value="seeder said no") + assert session_via_cli(["--no-seed", str(tmp_path)]).seeder.enabled is False + + +@pytest.fixture +def at_version(mocker: MockerFixture) -> Callable[[int, int], MagicMock]: + def build(major: int, minor: int) -> MagicMock: + interpreter = mocker.MagicMock() + interpreter.version_info = _VersionInfo(major, minor) + return interpreter + + return build + + +class _VersionInfo(NamedTuple): + major: int + minor: int diff --git a/tests/unit/seed/embed/test_bootstrap_link_via_app_data.py b/tests/unit/seed/embed/test_bootstrap_link_via_app_data.py index 4076573e1..70d38343b 100644 --- a/tests/unit/seed/embed/test_bootstrap_link_via_app_data.py +++ b/tests/unit/seed/embed/test_bootstrap_link_via_app_data.py @@ -3,17 +3,19 @@ import contextlib import os import sys +import zipfile from stat import S_IWGRP, S_IWOTH, S_IWUSR from subprocess import Popen, check_call from threading import Thread from typing import TYPE_CHECKING import pytest +from python_discovery import PythonInfo +from python_discovery import _cached_py_info as cached_py_info -from virtualenv.discovery import cached_py_info -from virtualenv.discovery.py_info import PythonInfo from virtualenv.info import fs_supports_symlink from virtualenv.run import cli_run +from virtualenv.seed.embed.via_app_data.pip_install.base import _safe_extract_zip from virtualenv.seed.wheels.embed import BUNDLE_FOLDER, BUNDLE_SUPPORT from virtualenv.util.path import safe_delete @@ -25,7 +27,7 @@ @pytest.mark.slow @pytest.mark.parametrize("copies", [False, True] if fs_supports_symlink() else [True]) -def test_seed_link_via_app_data(tmp_path, coverage_env, current_fastest, copies, for_py_version): # noqa: PLR0915 +def test_seed_link_via_app_data(tmp_path, coverage_env, current_fastest, copies) -> None: current = PythonInfo.current_system() bundle_ver = BUNDLE_SUPPORT[current.version_release_str] create_cmd = [ @@ -45,8 +47,6 @@ def test_seed_link_via_app_data(tmp_path, coverage_env, current_fastest, copies, current_fastest, "-vv", ] - if for_py_version == "3.8": - create_cmd += ["--wheel", bundle_ver["wheel"].split("-")[1]] if not copies: create_cmd.append("--symlink-app-data") result = cli_run(create_cmd) @@ -152,7 +152,7 @@ def read_only_app_data(temp_app_data): @pytest.mark.slow @pytest.mark.skipif(sys.platform == "win32", reason="Windows only applies R/O to files") @pytest.mark.usefixtures("read_only_app_data") -def test_base_bootstrap_link_via_app_data_not_writable(tmp_path, current_fastest): +def test_base_bootstrap_link_via_app_data_not_writable(tmp_path, current_fastest) -> None: dest = tmp_path / "venv" result = cli_run(["--seeder", "app-data", "--creator", current_fastest, "-vv", str(dest)]) assert result @@ -160,7 +160,7 @@ def test_base_bootstrap_link_via_app_data_not_writable(tmp_path, current_fastest @pytest.mark.slow @pytest.mark.skipif(sys.platform == "win32", reason="Windows only applies R/O to files") -def test_populated_read_only_cache_and_symlinked_app_data(tmp_path, current_fastest, temp_app_data): +def test_populated_read_only_cache_and_symlinked_app_data(tmp_path, current_fastest, temp_app_data) -> None: dest = tmp_path / "venv" cmd = [ "--seeder", @@ -175,7 +175,7 @@ def test_populated_read_only_cache_and_symlinked_app_data(tmp_path, current_fast assert cli_run(cmd) check_call((str(dest.joinpath("bin/python")), "-c", "import pip")) - cached_py_info._CACHE.clear() # necessary to re-trigger py info discovery # noqa: SLF001 + cached_py_info._CACHE.clear() # ruff:ignore[private-member-access] # necessary to re-trigger py info discovery safe_delete(dest) # should succeed with special flag when read-only @@ -186,7 +186,7 @@ def test_populated_read_only_cache_and_symlinked_app_data(tmp_path, current_fast @pytest.mark.slow @pytest.mark.skipif(sys.platform == "win32", reason="Windows only applies R/O to files") -def test_populated_read_only_cache_and_copied_app_data(tmp_path, current_fastest, temp_app_data): +def test_populated_read_only_cache_and_copied_app_data(tmp_path, current_fastest, temp_app_data) -> None: dest = tmp_path / "venv" cmd = [ "--seeder", @@ -201,7 +201,7 @@ def test_populated_read_only_cache_and_copied_app_data(tmp_path, current_fastest assert cli_run(cmd) - cached_py_info._CACHE.clear() # necessary to re-trigger py info discovery # noqa: SLF001 + cached_py_info._CACHE.clear() # ruff:ignore[private-member-access] # necessary to re-trigger py info discovery safe_delete(dest) # should succeed with special flag when read-only @@ -210,25 +210,18 @@ def test_populated_read_only_cache_and_copied_app_data(tmp_path, current_fastest @pytest.mark.slow -@pytest.mark.parametrize("pkg", ["pip", "setuptools", "wheel"]) +@pytest.mark.parametrize("pkg", ["pip", "setuptools"]) @pytest.mark.usefixtures("session_app_data", "current_fastest", "coverage_env") -def test_base_bootstrap_link_via_app_data_no(tmp_path, pkg, for_py_version): - if for_py_version != "3.8" and pkg == "wheel": - msg = "wheel isn't installed on Python > 3.8" - raise pytest.skip(msg) +def test_base_bootstrap_link_via_app_data_no(tmp_path, pkg) -> None: create_cmd = [str(tmp_path), "--seeder", "app-data", f"--no-{pkg}", "--setuptools", "bundle"] - if for_py_version == "3.8": - create_cmd += ["--wheel", "bundle"] result = cli_run(create_cmd) assert not (result.creator.purelib / pkg).exists() - for key in {"pip", "setuptools", "wheel"} - {pkg}: - if for_py_version != "3.8" and key == "wheel": - continue + for key in {"pip", "setuptools"} - {pkg}: assert (result.creator.purelib / key).exists() @pytest.mark.usefixtures("temp_app_data") -def test_app_data_parallel_ok(tmp_path): +def test_app_data_parallel_ok(tmp_path) -> None: exceptions = _run_parallel_threads(tmp_path) assert not exceptions, "\n".join(exceptions) @@ -246,13 +239,11 @@ def test_app_data_parallel_fail(tmp_path: Path, mocker: MockerFixture) -> None: def _run_parallel_threads(tmp_path): exceptions = [] - def _run(name): + def _run(name) -> None: try: cmd = ["--seeder", "app-data", str(tmp_path / name), "--no-setuptools"] - if sys.version_info[:2] == (3, 8): - cmd.append("--no-wheel") cli_run(cmd) - except Exception as exception: # noqa: BLE001 + except Exception as exception: # ruff:ignore[blind-except] as_str = str(exception) exceptions.append(as_str) @@ -262,3 +253,49 @@ def _run(name): for thread in threads: thread.join() return exceptions + + +def _write_zip_with_entry(zip_path: Path, entry_name: str) -> None: + with zipfile.ZipFile(str(zip_path), "w") as archive: + archive.writestr(entry_name, "payload") + + +def test_safe_extract_zip_allows_normal_entry(tmp_path: Path) -> None: + archive = tmp_path / "ok.zip" + _write_zip_with_entry(archive, "pkg/inner.txt") + target = tmp_path / "out" + target.mkdir() + + with zipfile.ZipFile(str(archive)) as zip_ref: + _safe_extract_zip(zip_ref, target) + + assert (target / "pkg" / "inner.txt").read_text(encoding="utf-8") == "payload" + + +def test_safe_extract_zip_rejects_parent_traversal(tmp_path: Path) -> None: + archive = tmp_path / "evil.zip" + _write_zip_with_entry(archive, "../escape.txt") + target = tmp_path / "out" + target.mkdir() + + with zipfile.ZipFile(str(archive)) as zip_ref, pytest.raises(RuntimeError, match="escaping target directory"): + _safe_extract_zip(zip_ref, target) + + assert not (tmp_path / "escape.txt").exists() + + +@pytest.mark.parametrize( + "entry", + [ + pytest.param("/tmp/evil.txt", id="posix"), # ruff:ignore[hardcoded-temp-file] + pytest.param("C:/Windows/System32/evil.txt", id="windows"), + ], +) +def test_safe_extract_zip_rejects_absolute_entry(tmp_path: Path, entry: str) -> None: + archive = tmp_path / "abs.zip" + _write_zip_with_entry(archive, entry) + target = tmp_path / "out" + target.mkdir() + + with zipfile.ZipFile(str(archive)) as zip_ref, pytest.raises(RuntimeError, match="absolute path"): + _safe_extract_zip(zip_ref, target) diff --git a/tests/unit/seed/embed/test_pip_invoke.py b/tests/unit/seed/embed/test_pip_invoke.py index 41f4395d2..730ec9eed 100644 --- a/tests/unit/seed/embed/test_pip_invoke.py +++ b/tests/unit/seed/embed/test_pip_invoke.py @@ -13,8 +13,8 @@ @pytest.mark.slow -@pytest.mark.parametrize("no", ["pip", "setuptools", "wheel", ""]) -def test_base_bootstrap_via_pip_invoke(tmp_path, coverage_env, mocker, current_fastest, no): # noqa: C901 +@pytest.mark.parametrize("no", ["pip", "setuptools", ""], ids=["no-pip", "no-setuptools", "all"]) +def test_base_bootstrap_via_pip_invoke(tmp_path, coverage_env, mocker, current_fastest, no) -> None: extra_search_dir = tmp_path / "extra" extra_search_dir.mkdir() for_py_version = f"{sys.version_info.major}.{sys.version_info.minor}" @@ -22,10 +22,10 @@ def test_base_bootstrap_via_pip_invoke(tmp_path, coverage_env, mocker, current_f for wheel_filename in BUNDLE_SUPPORT[for_py_version].values(): copy2(str(BUNDLE_FOLDER / wheel_filename), str(extra_search_dir)) - def _load_embed_wheel(app_data, distribution, for_py_version, version): # noqa: ARG001 + def _load_embed_wheel(app_data, distribution, _for_py_version, version): return load_embed_wheel(app_data, distribution, old_ver, version) - old_ver = "3.8" + old_ver = "3.9" old = BUNDLE_SUPPORT[old_ver] mocker.patch("virtualenv.seed.wheels.bundle.load_embed_wheel", side_effect=_load_embed_wheel) @@ -47,11 +47,9 @@ def _execute(cmd, env): assert found == expected_list return original(cmd, env) - original = PipInvoke._execute # noqa: SLF001 + original = PipInvoke._execute # ruff:ignore[private-member-access] run = mocker.patch.object(PipInvoke, "_execute", side_effect=_execute) versions = {"pip": "embed", "setuptools": "bundle"} - if sys.version_info[:2] == (3, 8): - versions["wheel"] = new["wheel"].split("-")[1] create_cmd = [ "--seeder", @@ -75,19 +73,5 @@ def _execute(cmd, env): assert result assert run.call_count == 1 - site_package = result.creator.purelib - pip = site_package / "pip" - setuptools = site_package / "setuptools" - wheel = site_package / "wheel" - files_post_first_create = list(site_package.iterdir()) - - if no: - no_file = locals()[no] - assert no not in files_post_first_create - - for key in ("pip", "setuptools", "wheel"): - if key == no: - continue - if sys.version_info[:2] >= (3, 9) and key == "wheel": - continue - assert locals()[key] in files_post_first_create + installed = {path.name for path in result.creator.purelib.iterdir()} + assert installed & {"pip", "setuptools"} == {"pip", "setuptools"} - {no} diff --git a/tests/unit/seed/wheels/test_acquire.py b/tests/unit/seed/wheels/test_acquire.py index 27b013d38..f5fb27a51 100644 --- a/tests/unit/seed/wheels/test_acquire.py +++ b/tests/unit/seed/wheels/test_acquire.py @@ -12,7 +12,7 @@ from virtualenv.app_data import AppDataDiskFolder from virtualenv.seed.wheels.acquire import download_wheel, get_wheel, pip_wheel_env_run from virtualenv.seed.wheels.embed import BUNDLE_FOLDER, get_embed_wheel -from virtualenv.seed.wheels.periodic_update import dump_datetime +from virtualenv.seed.wheels.periodic_update import UnverifiedWheelError, dump_datetime from virtualenv.seed.wheels.util import Wheel, discover_wheels if TYPE_CHECKING: @@ -23,21 +23,22 @@ @pytest.fixture(autouse=True) -def _fake_release_date(mocker): +def _fake_release_date(mocker) -> None: mocker.patch("virtualenv.seed.wheels.periodic_update.release_date_for_wheel_path", return_value=None) -def test_pip_wheel_env_run_could_not_find(session_app_data, mocker): +def test_pip_wheel_env_run_could_not_find(session_app_data, mocker) -> None: mocker.patch("virtualenv.seed.wheels.acquire.from_bundle", return_value=None) with pytest.raises(RuntimeError, match="could not find the embedded pip"): pip_wheel_env_run([], session_app_data, os.environ) -def test_download_wheel_bad_output(mocker, for_py_version, session_app_data): +def test_download_wheel_bad_output(mocker, for_py_version, session_app_data) -> None: """if the download contains no match for what wheel was downloaded, pick one that matches from target""" distribution = "setuptools" p_open = mocker.MagicMock() mocker.patch("virtualenv.seed.wheels.acquire.Popen", return_value=p_open) + mocker.patch("virtualenv.seed.wheels.acquire.verify_wheel_digest") p_open.communicate.return_value = "", "" p_open.returncode = 0 @@ -58,7 +59,20 @@ def test_download_wheel_bad_output(mocker, for_py_version, session_app_data): assert result.path == embed.path -def test_download_fails(mocker, for_py_version, session_app_data): +def test_download_wheel_not_found_raises(mocker, for_py_version, session_app_data) -> None: + """download_wheel must raise a clear error rather than crash when no wheel can be identified at all.""" + p_open = mocker.MagicMock() + mocker.patch("virtualenv.seed.wheels.acquire.Popen", return_value=p_open) + p_open.communicate.return_value = "", "" + p_open.returncode = 0 + as_path = mocker.MagicMock() + as_path.iterdir.return_value = [] + + with pytest.raises(RuntimeError, match="could not find downloaded wheel"): + download_wheel("setuptools", None, for_py_version, [], session_app_data, as_path, os.environ) + + +def test_download_fails(mocker, for_py_version, session_app_data) -> None: p_open = mocker.MagicMock() mocker.patch("virtualenv.seed.wheels.acquire.Popen", return_value=p_open) p_open.communicate.return_value = "out", "err" @@ -89,11 +103,12 @@ def test_download_fails(mocker, for_py_version, session_app_data): ] == exc.cmd -def test_download_wheel_python_io_encoding(mocker, for_py_version, session_app_data): +def test_download_wheel_python_io_encoding(mocker, for_py_version, session_app_data) -> None: mock_popen = mocker.patch("virtualenv.seed.wheels.acquire.Popen") mock_popen.return_value.communicate.return_value = "Saved a-b-c.whl", "" mock_popen.return_value.returncode = 0 mocker.patch("pathlib.Path.absolute", return_value=Path("a-b-c.whl")) + mocker.patch("virtualenv.seed.wheels.acquire.verify_wheel_digest") download_wheel("pip", "==1", for_py_version, [], session_app_data, "folder", os.environ.copy()) @@ -101,6 +116,43 @@ def test_download_wheel_python_io_encoding(mocker, for_py_version, session_app_d assert env["PYTHONIOENCODING"] == "utf-8" +def test_download_wheel_verifies_digest(mocker, for_py_version, session_app_data) -> None: + """download_wheel must reject a wheel whose bytes do not match PyPI's published digest.""" + mock_popen = mocker.patch("virtualenv.seed.wheels.acquire.Popen") + mock_popen.return_value.communicate.return_value = "Saved a-b-c.whl", "" + mock_popen.return_value.returncode = 0 + mocker.patch("pathlib.Path.absolute", return_value=Path("a-b-c.whl")) + verify = mocker.patch( + "virtualenv.seed.wheels.acquire.verify_wheel_digest", + side_effect=RuntimeError("digest mismatch"), + ) + + with pytest.raises(RuntimeError, match="digest mismatch"): + download_wheel("pip", "==1", for_py_version, [], session_app_data, "folder", os.environ.copy()) + + assert verify.call_count == 1 + + +@pytest.mark.parametrize("env_var", ["PIP_INDEX_URL", "PIP_EXTRA_INDEX_URL", "PIP_INDEX"]) +def test_download_wheel_skips_digest_check_for_custom_index(mocker, for_py_version, session_app_data, env_var) -> None: + """A configured custom index may legitimately serve a same-named wheel PyPI never built.""" + mock_popen = mocker.patch("virtualenv.seed.wheels.acquire.Popen") + mock_popen.return_value.communicate.return_value = "Saved a-b-c.whl", "" + mock_popen.return_value.returncode = 0 + mocker.patch("pathlib.Path.absolute", return_value=Path("a-b-c.whl")) + verify = mocker.patch( + "virtualenv.seed.wheels.acquire.verify_wheel_digest", + side_effect=RuntimeError("would have rejected a legitimate wheel"), + ) + env = os.environ.copy() + env[env_var] = "https://example.com/simple" + + result = download_wheel("pip", "==1", for_py_version, [], session_app_data, "folder", env) + + assert result.name == "a-b-c.whl" + assert verify.call_count == 0 + + @pytest.fixture def downloaded_wheel(mocker): wheel = Wheel.from_path(Path("setuptools-0.0.0-py2.py3-none-any.whl")) @@ -108,7 +160,7 @@ def downloaded_wheel(mocker): @pytest.mark.parametrize("version", ["bundle", "0.0.0"]) -def test_get_wheel_download_called(mocker, for_py_version, session_app_data, downloaded_wheel, version): +def test_get_wheel_download_called(mocker, for_py_version, session_app_data, downloaded_wheel, version) -> None: distribution = "setuptools" write = mocker.patch("virtualenv.app_data.via_disk_folder.JSONStoreDisk.write") wheel = get_wheel(distribution, version, for_py_version, [], True, session_app_data, False, os.environ) @@ -119,7 +171,7 @@ def test_get_wheel_download_called(mocker, for_py_version, session_app_data, dow @pytest.mark.parametrize("version", ["embed", "pinned"]) -def test_get_wheel_download_not_called(mocker, for_py_version, session_app_data, downloaded_wheel, version): +def test_get_wheel_download_not_called(mocker, for_py_version, session_app_data, downloaded_wheel, version) -> None: distribution = "setuptools" expected = get_embed_wheel(distribution, for_py_version) if version == "pinned": @@ -132,6 +184,26 @@ def test_get_wheel_download_not_called(mocker, for_py_version, session_app_data, assert write.call_count == 0 +def test_get_wheel_falls_back_to_bundle_when_unverified( + mocker: MockerFixture, for_py_version: str, session_app_data: AppDataDiskFolder +) -> None: + mocker.patch("virtualenv.seed.wheels.acquire.download_wheel", side_effect=UnverifiedWheelError("refusing")) + + wheel = get_wheel("setuptools", "bundle", for_py_version, [], True, session_app_data, False, os.environ) + + assert wheel is not None + assert wheel.name == get_embed_wheel("setuptools", for_py_version).name + + +def test_get_wheel_unverified_pinned_version_fails( + mocker: MockerFixture, for_py_version: str, session_app_data: AppDataDiskFolder +) -> None: + mocker.patch("virtualenv.seed.wheels.acquire.download_wheel", side_effect=UnverifiedWheelError("refusing")) + + with pytest.raises(UnverifiedWheelError, match="refusing"): + get_wheel("setuptools", "0.0.0", for_py_version, [], True, session_app_data, False, os.environ) + + def test_get_wheel_download_cached( tmp_path: Path, mocker: MockerFixture, @@ -140,7 +212,7 @@ def test_get_wheel_download_cached( time_freeze: Callable[[datetime], None], ) -> None: time_freeze(datetime.now(tz=timezone.utc)) - from virtualenv.app_data.via_disk_folder import JSONStoreDisk # noqa: PLC0415 + from virtualenv.app_data.via_disk_folder import JSONStoreDisk # ruff:ignore[import-outside-top-level] app_data = AppDataDiskFolder(folder=str(tmp_path)) expected = downloaded_wheel[0] @@ -171,3 +243,36 @@ def test_get_wheel_download_cached( }, ], } + + +@pytest.mark.parametrize( + "distribution", + [ + "pip space", + "pip[extra]", + "pip;os.system('x')", + "--index-url=http://evil", + "-r requirements.txt", + "", + ".pip", + "pip-", + ], +) +def test_download_wheel_rejects_bad_distribution(distribution: str, session_app_data) -> None: + with pytest.raises(ValueError, match="suspicious distribution name"): + download_wheel(distribution, None, "3.14", [], session_app_data, "folder", os.environ) + + +@pytest.mark.parametrize( + "version_spec", + [ + "==1.0 --index-url=http://evil", + ">=1.0", + "== 1.0", + "==1.0;echo", + ], +) +def test_download_wheel_rejects_bad_version_spec(version_spec: str, session_app_data, mocker: MockerFixture) -> None: + mocker.patch("virtualenv.seed.wheels.acquire.Popen") + with pytest.raises(ValueError, match="suspicious version spec"): + download_wheel("pip", version_spec, "3.14", [], session_app_data, "folder", os.environ) diff --git a/tests/unit/seed/wheels/test_acquire_find_wheel.py b/tests/unit/seed/wheels/test_acquire_find_wheel.py index 7822849e5..b3dd1b506 100644 --- a/tests/unit/seed/wheels/test_acquire_find_wheel.py +++ b/tests/unit/seed/wheels/test_acquire_find_wheel.py @@ -1,29 +1,116 @@ from __future__ import annotations +import zipfile +from typing import TYPE_CHECKING, Final + import pytest from virtualenv.seed.wheels.acquire import find_compatible_in_house from virtualenv.seed.wheels.embed import BUNDLE_FOLDER, MAX, get_embed_wheel +from virtualenv.seed.wheels.util import Wheel, discover_wheels + +if TYPE_CHECKING: + from collections.abc import Callable + from pathlib import Path -def test_find_latest_none(for_py_version): - result = find_compatible_in_house("setuptools", None, for_py_version, BUNDLE_FOLDER) - expected = get_embed_wheel("setuptools", for_py_version) +@pytest.mark.parametrize("version", [pytest.param(None, id="none"), pytest.param("", id="empty")]) +def test_find_latest(for_py_version: str, version: str | None) -> None: + result: Final[Wheel | None] = find_compatible_in_house("setuptools", version, for_py_version, BUNDLE_FOLDER) + expected: Final[Wheel | None] = get_embed_wheel("setuptools", for_py_version) + assert result is not None + assert expected is not None assert result.path == expected.path -def test_find_latest_string(for_py_version): - result = find_compatible_in_house("setuptools", "", for_py_version, BUNDLE_FOLDER) - expected = get_embed_wheel("setuptools", for_py_version) +def test_find_exact(for_py_version: str) -> None: + expected: Final[Wheel | None] = get_embed_wheel("setuptools", for_py_version) + assert expected is not None + result: Final[Wheel | None] = find_compatible_in_house( + "setuptools", f"=={expected.version}", for_py_version, BUNDLE_FOLDER + ) + assert result is not None assert result.path == expected.path -def test_find_exact(for_py_version): - expected = get_embed_wheel("setuptools", for_py_version) - result = find_compatible_in_house("setuptools", f"=={expected.version}", for_py_version, BUNDLE_FOLDER) - assert result.path == expected.path +@pytest.mark.parametrize("spec", [pytest.param("bad", id="invalid"), pytest.param(">=1", id="unsupported")]) +def test_find_bad_spec(spec: str) -> None: + with pytest.raises(ValueError, match=spec): + find_compatible_in_house("setuptools", spec, MAX, BUNDLE_FOLDER) + + +@pytest.mark.parametrize( + ("other", "requested"), + [ + pytest.param("26.2", "26.2.1rc1", id="prerelease"), + pytest.param("26.2.1rc1", "26.2.1rc1.dev2", id="prerelease-dev"), + pytest.param("26.2.1.post1", "26.2.1.post1.dev2", id="postrelease-dev"), + pytest.param("26.2.1", "26.2.1.1", id="four-release-components"), + pytest.param("26.2.1", "26.2.1+local.1", id="local"), + pytest.param("1!26.2.1", "1!26.2.2", id="epoch"), + ], +) +@pytest.mark.parametrize("available", [True, False], ids=["present", "missing"]) +def test_find_exact_version( + tmp_path: Path, make_wheel: Callable[[str], Path], other: str, requested: str, available: bool +) -> None: + make_wheel(other) + expected: Final[Path | None] = make_wheel(requested) if available else None + result: Final[Wheel | None] = find_compatible_in_house("pip", f"=={requested}", "3.14", tmp_path) + assert (result.path if result else None) == expected + + +@pytest.mark.parametrize( + ("older", "newer"), + [ + pytest.param("26.2", "26.2.1rc1", id="release-before-next-prerelease"), + pytest.param("26.2.1rc1", "26.2.1", id="prerelease-before-final"), + pytest.param("26.2.1rc1.dev1", "26.2.1rc1", id="prerelease-dev"), + pytest.param("26.2.1.post1.dev1", "26.2.1.post1", id="postrelease-dev"), + pytest.param("26.2.1", "26.2.1.1", id="four-release-components"), + pytest.param("26.2.1+abc", "26.2.1+1", id="local-numeric-after-text"), + pytest.param("26.2.1", "1!1.0", id="epoch"), + ], +) +def test_discover_version_order(tmp_path: Path, make_wheel: Callable[[str], Path], older: str, newer: str) -> None: + make_wheel(older) + make_wheel(newer) + assert [wheel.version for wheel in discover_wheels(tmp_path, "pip", None, "3.14")] == [newer, older] + + +@pytest.mark.parametrize("version", ["26.2.1", "26.2.1rc1", "26.2.1.post1", "26.2.1.dev1"]) +def test_version_tuple_contains_release_numbers(make_wheel: Callable[[str], Path], version: str) -> None: + assert Wheel(make_wheel(version)).version_tuple == (26, 2, 1) + + +@pytest.mark.parametrize( + ("versions", "spec", "expected"), + [ + pytest.param(("26.2.0", "26.2.1rc1"), "<26.2.1", "26.2.0", id="exclude-bound-prerelease"), + pytest.param(("26.2.0", "26.2.1rc1"), "<26.2.1rc2", "26.2.1rc1", id="explicit-prerelease-bound"), + pytest.param(("26.2.1", "26.2.1+local.1"), "==26.2.1", "26.2.1+local.1", id="public-pin"), + pytest.param(("26.2.1", "26.2.1rc1"), "==26.2.1RC1", "26.2.1rc1", id="normalized-pin"), + ], +) +def test_find_version_specifier( + tmp_path: Path, make_wheel: Callable[[str], Path], versions: tuple[str, ...], spec: str, expected: str +) -> None: + for version in versions: + make_wheel(version) + result: Final[Wheel | None] = find_compatible_in_house("pip", spec, "3.14", tmp_path) + assert result is not None + assert result.version == expected + +@pytest.fixture +def make_wheel(tmp_path: Path) -> Callable[[str], Path]: + def build(version: str) -> Path: + path: Final[Path] = tmp_path / f"pip-{version}-py3-none-any.whl" + with zipfile.ZipFile(path, "w") as wheel: + wheel.writestr( + f"pip-{version}.dist-info/METADATA", + f"Metadata-Version: 2.1\nName: pip\nVersion: {version}\nRequires-Python: >=3.9\n", + ) + return path -def test_find_bad_spec(): - with pytest.raises(ValueError, match="bad"): - find_compatible_in_house("setuptools", "bad", MAX, BUNDLE_FOLDER) + return build diff --git a/tests/unit/seed/wheels/test_bundle.py b/tests/unit/seed/wheels/test_bundle.py index d0e95cf31..41dd7b9d0 100644 --- a/tests/unit/seed/wheels/test_bundle.py +++ b/tests/unit/seed/wheels/test_bundle.py @@ -1,14 +1,23 @@ from __future__ import annotations +import hashlib import os +import zipfile from datetime import datetime, timezone from pathlib import Path import pytest from virtualenv.app_data import AppDataDiskFolder +from virtualenv.seed.wheels import embed from virtualenv.seed.wheels.bundle import from_bundle -from virtualenv.seed.wheels.embed import get_embed_wheel +from virtualenv.seed.wheels.embed import ( + BUNDLE_FOLDER, + BUNDLE_SHA256, + BUNDLE_SUPPORT, + _verify_bundled_wheel, + get_embed_wheel, +) from virtualenv.seed.wheels.periodic_update import dump_datetime from virtualenv.seed.wheels.util import Version, Wheel @@ -45,31 +54,95 @@ def app_data(tmp_path_factory, for_py_version, next_pip_wheel): return app_data_ -def test_version_embed(app_data, for_py_version): +def test_version_embed(app_data, for_py_version) -> None: wheel = from_bundle("pip", Version.embed, for_py_version, [], app_data, False, os.environ) assert wheel is not None assert wheel.name == get_embed_wheel("pip", for_py_version).name -def test_version_bundle(app_data, for_py_version, next_pip_wheel): +def test_version_bundle(app_data, for_py_version, next_pip_wheel) -> None: wheel = from_bundle("pip", Version.bundle, for_py_version, [], app_data, False, os.environ) assert wheel is not None assert wheel.name == next_pip_wheel.name -def test_version_pinned_not_found(app_data, for_py_version): +def test_version_pinned_not_found(app_data, for_py_version) -> None: wheel = from_bundle("pip", "0.0.0", for_py_version, [], app_data, False, os.environ) assert wheel is None -def test_version_pinned_is_embed(app_data, for_py_version): +def test_version_pinned_is_embed(app_data, for_py_version) -> None: expected_wheel = get_embed_wheel("pip", for_py_version) wheel = from_bundle("pip", expected_wheel.version, for_py_version, [], app_data, False, os.environ) assert wheel is not None assert wheel.name == expected_wheel.name -def test_version_pinned_in_app_data(app_data, for_py_version, next_pip_wheel): +def test_version_pinned_in_app_data(app_data, for_py_version, next_pip_wheel) -> None: wheel = from_bundle("pip", next_pip_wheel.version, for_py_version, [], app_data, False, os.environ) assert wheel is not None assert wheel.name == next_pip_wheel.name + + +def test_every_bundled_wheel_has_sha256() -> None: + referenced = {wheel for mapping in BUNDLE_SUPPORT.values() for wheel in mapping.values()} + missing = referenced - BUNDLE_SHA256.keys() + assert not missing, f"bundled wheels missing from BUNDLE_SHA256: {sorted(missing)}" + + +def test_every_wheel_on_disk_has_sha256() -> None: + on_disk = {entry.name for entry in BUNDLE_FOLDER.iterdir() if entry.suffix == ".whl"} + assert on_disk == BUNDLE_SHA256.keys() + + +def test_notices_mention_every_bundled_distribution() -> None: + # rooted at this test file rather than BUNDLE_FOLDER: a non-editable install puts the installed package + # under site-packages with no relationship to the checkout, but tests/ always runs from the repo itself + repo_root = Path(__file__).resolve().parents[4] + notices = (repo_root / "THIRD-PARTY-NOTICES.md").read_text(encoding="utf-8") + distributions = {wheel_name.split("-")[0] for wheel_name in BUNDLE_SHA256} + missing = {name for name in distributions if f"## {name}" not in notices} + assert not missing, f"THIRD-PARTY-NOTICES.md is missing a section for: {sorted(missing)}" + + +def test_get_embed_wheel_verifies_pip(for_py_version: str) -> None: + wheel = get_embed_wheel("pip", for_py_version) + assert wheel is not None + + +def test_verify_bundled_wheel_rejects_tamper(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + fake_name = "fake-0.0.1-py3-none-any.whl" + fake = tmp_path / fake_name + fake.write_bytes(b"not the real bytes") + monkeypatch.setitem(BUNDLE_SHA256, fake_name, "0" * 64) + monkeypatch.setattr("virtualenv.seed.wheels.embed._VERIFIED_WHEELS", set()) + + with pytest.raises(RuntimeError, match="sha256 mismatch"): + _verify_bundled_wheel(fake) + + +def test_verify_bundled_wheel_rejects_unknown(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + stray = tmp_path / "stray-0.0.1-py3-none-any.whl" + stray.write_bytes(b"payload") + monkeypatch.setattr("virtualenv.seed.wheels.embed._VERIFIED_WHEELS", set()) + + with pytest.raises(RuntimeError, match="no recorded sha256"): + _verify_bundled_wheel(stray) + + +def test_verify_bundled_wheel_reads_from_zipapp(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + # Simulate the layout we get when virtualenv runs out of a ``.pyz``: the wheel lives inside the zipapp rather + # than on disk, so the hash check has to read it through ``zipfile`` rather than ``path.open``. + wheel_name = "fakepkg-0.0.1-py3-none-any.whl" + wheel_payload = b"pretend-wheel-bytes" + fake_root = tmp_path / "virtualenv.pyz" + entry = f"virtualenv/seed/wheels/embed/{wheel_name}" + with zipfile.ZipFile(str(fake_root), "w") as archive: + archive.writestr(entry, wheel_payload) + + monkeypatch.setattr(embed, "IS_ZIPAPP", True) + monkeypatch.setattr(embed, "ROOT", str(fake_root)) + monkeypatch.setitem(BUNDLE_SHA256, wheel_name, hashlib.sha256(wheel_payload).hexdigest()) + monkeypatch.setattr(embed, "_VERIFIED_WHEELS", set()) + + _verify_bundled_wheel(fake_root / entry) diff --git a/tests/unit/seed/wheels/test_periodic_update.py b/tests/unit/seed/wheels/test_periodic_update.py index e11fe5e1a..ae21622ff 100644 --- a/tests/unit/seed/wheels/test_periodic_update.py +++ b/tests/unit/seed/wheels/test_periodic_update.py @@ -1,6 +1,8 @@ from __future__ import annotations +import hashlib import json +import logging import os import subprocess import sys @@ -11,6 +13,7 @@ from itertools import zip_longest from pathlib import Path from textwrap import dedent +from typing import TYPE_CHECKING, Final from urllib.error import URLError import pytest @@ -21,26 +24,44 @@ from virtualenv.seed.wheels.embed import BUNDLE_SUPPORT, get_embed_wheel from virtualenv.seed.wheels.periodic_update import ( NewVersion, + UnverifiedWheelError, UpdateLog, + add_wheel_to_update_log, do_update, dump_datetime, - load_datetime, manual_upgrade, periodic_update, release_date_for_wheel_path, trigger_update, + verify_wheel_digest, ) from virtualenv.util.subprocess import CREATE_NO_WINDOW +if TYPE_CHECKING: + from collections.abc import Callable, Generator + from unittest.mock import MagicMock + + from pytest_mock import MockerFixture + + from virtualenv.app_data.via_disk_folder import EmbedDistributionUpdateStoreDisk + @pytest.fixture(autouse=True) -def _clear_pypi_info_cache(): - from virtualenv.seed.wheels.periodic_update import _PYPI_CACHE # noqa: PLC0415 +def _clear_pypi_info_cache() -> Generator[None, None, None]: + from virtualenv.seed.wheels.periodic_update import _PYPI_CACHE # ruff:ignore[import-outside-top-level] _PYPI_CACHE.clear() + yield + # the mocked responses must not leak into later tests that verify real downloads in this process + _PYPI_CACHE.clear() + + +@pytest.fixture +def retry_sleep(mocker: MockerFixture) -> MagicMock: + return mocker.patch("virtualenv.seed.wheels.periodic_update.sleep") -def test_manual_upgrade(session_app_data, caplog, mocker, for_py_version): +def test_manual_upgrade(session_app_data, caplog, mocker, for_py_version) -> None: wheel = get_embed_wheel("pip", for_py_version) new_version = NewVersion( wheel.path, @@ -49,14 +70,7 @@ def test_manual_upgrade(session_app_data, caplog, mocker, for_py_version): "manual", ) - def _do_update( # noqa: PLR0913 - distribution, - for_py_version, # noqa: ARG001 - embed_filename, # noqa: ARG001 - app_data, # noqa: ARG001 - search_dirs, # noqa: ARG001 - periodic, # noqa: ARG001 - ): + def _do_update(distribution, **_kwargs): if distribution == "pip": return [new_version] return [] @@ -74,12 +88,12 @@ def _do_update( # noqa: PLR0913 packages[args[1]["distribution"]].append(args[1]["for_py_version"]) packages = {key: sorted(value) for key, value in packages.items()} versions = sorted(BUNDLE_SUPPORT.keys()) - expected = {"setuptools": versions, "wheel": ["3.8"], "pip": versions} + expected = {"setuptools": versions, "pip": versions} assert packages == expected @pytest.mark.usefixtures("session_app_data") -def test_pick_periodic_update(tmp_path, mocker, for_py_version): +def test_pick_periodic_update(tmp_path, mocker, for_py_version) -> None: embed, current = get_embed_wheel("setuptools", "3.6"), get_embed_wheel("setuptools", for_py_version) mocker.patch("virtualenv.seed.wheels.bundle.load_embed_wheel", return_value=embed) completed = datetime.now(tz=timezone.utc) - timedelta(days=29) @@ -109,7 +123,7 @@ def test_pick_periodic_update(tmp_path, mocker, for_py_version): assert f"setuptools-{current.version}.dist-info" in installed -def test_periodic_update_stops_at_current(mocker, session_app_data, for_py_version): +def test_periodic_update_stops_at_current(mocker, session_app_data, for_py_version) -> None: current = get_embed_wheel("setuptools", for_py_version) now, completed = datetime.now(tz=timezone.utc), datetime.now(tz=timezone.utc) - timedelta(days=29) @@ -129,7 +143,7 @@ def test_periodic_update_stops_at_current(mocker, session_app_data, for_py_versi assert result.path == current.path -def test_periodic_update_latest_per_patch(mocker, session_app_data, for_py_version): +def test_periodic_update_latest_per_patch(mocker, session_app_data, for_py_version) -> None: current = get_embed_wheel("setuptools", for_py_version) expected_path = wheel_path(current, (0, 1, 2)) now = datetime.now(tz=timezone.utc) @@ -150,7 +164,7 @@ def test_periodic_update_latest_per_patch(mocker, session_app_data, for_py_versi assert str(result.path) == expected_path -def test_periodic_update_latest_per_patch_prev_is_manual(mocker, session_app_data, for_py_version): +def test_periodic_update_latest_per_patch_prev_is_manual(mocker, session_app_data, for_py_version) -> None: current = get_embed_wheel("setuptools", for_py_version) expected_path = wheel_path(current, (0, 1, 2)) now = datetime.now(tz=timezone.utc) @@ -172,7 +186,7 @@ def test_periodic_update_latest_per_patch_prev_is_manual(mocker, session_app_dat assert str(result.path) == expected_path -def test_manual_update_honored(mocker, session_app_data, for_py_version): +def test_manual_update_honored(mocker, session_app_data, for_py_version) -> None: current = get_embed_wheel("setuptools", for_py_version) expected_path = wheel_path(current, (0, 1, 1)) now = datetime.now(tz=timezone.utc) @@ -200,8 +214,8 @@ def wheel_path(wheel, of, pre_release=""): return str(wheel.path.parent / new_name) -_UP_NOW = datetime.now(tz=timezone.utc) -_UPDATE_SKIP = { +_UP_NOW: Final[datetime] = datetime.now(tz=timezone.utc) +_UPDATE_SKIP: Final[dict[str, UpdateLog]] = { "started_just_now_no_complete": UpdateLog(started=_UP_NOW, completed=None, versions=[], periodic=True), "started_1_hour_no_complete": UpdateLog( started=_UP_NOW - timedelta(hours=1), @@ -231,16 +245,129 @@ def wheel_path(wheel, of, pre_release=""): @pytest.mark.parametrize("u_log", list(_UPDATE_SKIP.values()), ids=list(_UPDATE_SKIP.keys())) -def test_periodic_update_skip(u_log, mocker, for_py_version, session_app_data, time_freeze): +@pytest.mark.usefixtures("periodic_update_clock") +def test_periodic_update_skip( + u_log: UpdateLog, + mocker: MockerFixture, + for_py_version: str, + app_data: AppDataDiskFolder, +) -> None: + app_data.embed_update_log("setuptools", for_py_version).write(u_log.to_dict()) + mocker.patch("virtualenv.seed.wheels.periodic_update.Popen", autospec=True, side_effect=AssertionError) + + assert ( + periodic_update("setuptools", None, for_py_version, None, [], app_data, do_periodic_update=True, env={}), + app_data.embed_update_log("setuptools", for_py_version).read(), + ) == (None, u_log.to_dict()) + + +@pytest.fixture +def app_data(tmp_path: Path) -> AppDataDiskFolder: + return AppDataDiskFolder(str(tmp_path)) + + +@pytest.fixture +def periodic_update_clock(time_freeze: Callable[[datetime], None]) -> None: time_freeze(_UP_NOW) - mocker.patch("virtualenv.app_data.via_disk_folder.JSONStoreDisk.read", return_value=u_log.to_dict()) - mocker.patch("virtualenv.seed.wheels.periodic_update.trigger_update", side_effect=RuntimeError) - result = periodic_update("setuptools", None, for_py_version, None, [], session_app_data, os.environ, True) - assert result is None + +def test_periodic_update_drops_malformed_log( + malformed_app_data: AppDataDiskFolder, + for_py_version: str, + caplog: pytest.LogCaptureFixture, +) -> None: + caplog.set_level(logging.WARNING) + wheel: Final[Wheel | None] = get_embed_wheel("pip", for_py_version) + + result: Final[Wheel | None] = periodic_update( + "pip", None, for_py_version, wheel, [], malformed_app_data, False, os.environ + ) + + assert (result, malformed_app_data.embed_update_log("pip", for_py_version).exists(), caplog.messages) == ( + wheel, + False, + ["removing malformed embed update log of pip"], + ) + + +def test_periodic_update_auto_update_drops_malformed_log( + malformed_app_data: AppDataDiskFolder, + for_py_version: str, + caplog: pytest.LogCaptureFixture, + mocker: MockerFixture, +) -> None: + caplog.set_level(logging.WARNING) + trigger_update_: Final[MagicMock] = mocker.patch("virtualenv.seed.wheels.periodic_update.trigger_update") + wheel: Final[Wheel | None] = get_embed_wheel("pip", for_py_version) + + result: Final[Wheel | None] = periodic_update( + "pip", None, for_py_version, wheel, [], malformed_app_data, True, os.environ + ) + + log: Final[EmbedDistributionUpdateStoreDisk] = malformed_app_data.embed_update_log("pip", for_py_version) + # a reset log never completed an update, so handle_auto_update schedules one + assert (result, trigger_update_.call_count, log.read()["periodic"], caplog.messages) == ( + wheel, + 1, + True, + ["removing malformed embed update log of pip"], + ) + + +def test_add_wheel_to_update_log_drops_malformed_log( + malformed_app_data: AppDataDiskFolder, + for_py_version: str, + caplog: pytest.LogCaptureFixture, + time_freeze: Callable[[datetime], None], +) -> None: + caplog.set_level(logging.WARNING) + time_freeze(_UP_NOW) + wheel: Final[Wheel] = get_embed_wheel("pip", for_py_version) + + add_wheel_to_update_log(wheel, for_py_version, malformed_app_data) + + assert (malformed_app_data.embed_update_log("pip", for_py_version).read()["versions"], caplog.messages) == ( + [NewVersion(wheel.path.name, _UP_NOW, None, "download").to_dict()], + ["removing malformed embed update log of pip"], + ) + + +def test_do_update_drops_malformed_log( + malformed_app_data: AppDataDiskFolder, + for_py_version: str, + caplog: pytest.LogCaptureFixture, + mocker: MockerFixture, + time_freeze: Callable[[datetime], None], +) -> None: + caplog.set_level(logging.WARNING) + time_freeze(_UP_NOW) + mocker.patch("virtualenv.seed.wheels.acquire.download_wheel", return_value=None) + + versions: Final[list[NewVersion] | None] = do_update("pip", for_py_version, None, malformed_app_data, [], True) + + assert (versions, malformed_app_data.embed_update_log("pip", for_py_version).read(), caplog.messages) == ( + [], + UpdateLog(None, _UP_NOW, [], True).to_dict(), + ["removing malformed embed update log of pip"], + ) + + +@pytest.fixture( + params=[ + pytest.param([1, 2, 3], id="not-a-mapping"), + pytest.param("oops", id="string"), + pytest.param({"versions": "abc"}, id="versions-not-a-list"), + pytest.param({"versions": ["abc"]}, id="version-not-a-mapping"), + pytest.param({"completed": 5, "started": None, "versions": []}, id="completed-not-a-datetime"), + ] +) +def malformed_app_data(request: pytest.FixtureRequest, tmp_path: Path, for_py_version: str) -> AppDataDiskFolder: + app_data: Final[AppDataDiskFolder] = AppDataDiskFolder(str(tmp_path)) + app_data.embed_update_log("pip", for_py_version).write(request.param) + return app_data -_UPDATE_YES = { +_UPDATE_YES: Final[dict[str, UpdateLog]] = { "never_started": UpdateLog(started=None, completed=None, versions=[], periodic=False), "started_1_hour": UpdateLog( started=_UP_NOW - timedelta(hours=1, microseconds=1), @@ -258,29 +385,44 @@ def test_periodic_update_skip(u_log, mocker, for_py_version, session_app_data, t @pytest.mark.parametrize("u_log", list(_UPDATE_YES.values()), ids=list(_UPDATE_YES.keys())) -def test_periodic_update_trigger(u_log, mocker, for_py_version, session_app_data, time_freeze): - time_freeze(_UP_NOW) - mocker.patch("virtualenv.app_data.via_disk_folder.JSONStoreDisk.read", return_value=u_log.to_dict()) - write = mocker.patch("virtualenv.app_data.via_disk_folder.JSONStoreDisk.write") - trigger_update_ = mocker.patch("virtualenv.seed.wheels.periodic_update.trigger_update") - - result = periodic_update("setuptools", None, for_py_version, None, [], session_app_data, os.environ, True) +@pytest.mark.usefixtures("periodic_update_clock") +@pytest.mark.parametrize( + "env", + [pytest.param({}, id="empty-env"), pytest.param({"_VIRTUALENV_PERIODIC_UPDATE_INLINE": "1"}, id="custom-env")], +) +def test_periodic_update_trigger( + u_log: UpdateLog, + mocker: MockerFixture, + for_py_version: str, + app_data: AppDataDiskFolder, + env: dict[str, str], +) -> None: + app_data.embed_update_log("setuptools", for_py_version).write(u_log.to_dict()) + process: Final[MagicMock] = mocker.create_autospec(subprocess.Popen, instance=True, pid=123) + popen: Final[MagicMock] = mocker.patch( + "virtualenv.seed.wheels.periodic_update.Popen", autospec=True, return_value=process + ) - assert result is None - assert trigger_update_.call_count - assert write.call_count == 1 - wrote_json = write.call_args[0][0] - assert wrote_json["periodic"] is True - assert load_datetime(wrote_json["started"]) == _UP_NOW + assert ( + periodic_update("setuptools", None, for_py_version, None, [], app_data, do_periodic_update=True, env=env), + app_data.embed_update_log("setuptools", for_py_version).read(), + ) == (None, UpdateLog(_UP_NOW, u_log.completed, [], True).to_dict()) + assert popen.call_count == 1 + assert popen.call_args.kwargs == { + "stdout": None if env else subprocess.DEVNULL, + "stderr": None if env else subprocess.DEVNULL, + **({"creationflags": CREATE_NO_WINDOW} if not env and sys.platform == "win32" else {}), + } + assert process.communicate.call_count == bool(env) -def test_trigger_update_no_debug(for_py_version, session_app_data, tmp_path, mocker, monkeypatch): +def test_trigger_update_no_debug(for_py_version, session_app_data, tmp_path, mocker, monkeypatch) -> None: monkeypatch.delenv("_VIRTUALENV_PERIODIC_UPDATE_INLINE", raising=False) current = get_embed_wheel("setuptools", for_py_version) process = mocker.MagicMock() process.pid = 123 process.communicate.return_value = None, None - Popen = mocker.patch("virtualenv.seed.wheels.periodic_update.Popen", return_value=process) # noqa: N806 + Popen = mocker.patch("virtualenv.seed.wheels.periodic_update.Popen", return_value=process) # ruff:ignore[non-lowercase-variable-in-function] trigger_update( "setuptools", @@ -322,14 +464,14 @@ def test_trigger_update_no_debug(for_py_version, session_app_data, tmp_path, moc assert process.communicate.call_count == 0 -def test_trigger_update_debug(for_py_version, session_app_data, tmp_path, mocker, monkeypatch): +def test_trigger_update_debug(for_py_version, session_app_data, tmp_path, mocker, monkeypatch) -> None: monkeypatch.setenv("_VIRTUALENV_PERIODIC_UPDATE_INLINE", "1") current = get_embed_wheel("pip", for_py_version) process = mocker.MagicMock() process.pid = 123 process.communicate.return_value = None, None - Popen = mocker.patch("virtualenv.seed.wheels.periodic_update.Popen", return_value=process) # noqa: N806 + Popen = mocker.patch("virtualenv.seed.wheels.periodic_update.Popen", return_value=process) # ruff:ignore[non-lowercase-variable-in-function] trigger_update( "pip", @@ -368,7 +510,7 @@ def test_trigger_update_debug(for_py_version, session_app_data, tmp_path, mocker assert process.communicate.call_count == 1 -def test_do_update_first(tmp_path, mocker, time_freeze): +def test_do_update_first(tmp_path, mocker, time_freeze) -> None: time_freeze(_UP_NOW) wheel = get_embed_wheel("pip", "3.9") app_data_outer = AppDataDiskFolder(str(tmp_path / "app")) @@ -384,14 +526,14 @@ def test_do_update_first(tmp_path, mocker, time_freeze): ] download_wheels = (Wheel(Path(i[0])) for i in pip_version_remote) - def _download_wheel( # noqa: PLR0913 + def _download_wheel( # ruff:ignore[too-many-arguments] distribution, - version_spec, # noqa: ARG001 + version_spec, # ruff:ignore[unused-function-argument] for_py_version, search_dirs, app_data, to_folder, - env, # noqa: ARG001 + env, # ruff:ignore[unused-function-argument] ): assert distribution == "pip" assert for_py_version == "3.9" @@ -446,22 +588,14 @@ def _release(of, context): } -def test_do_update_skip_already_done(tmp_path, mocker, time_freeze): +def test_do_update_skip_already_done(tmp_path, mocker, time_freeze) -> None: time_freeze(_UP_NOW + timedelta(hours=1)) wheel = get_embed_wheel("pip", "3.9") app_data_outer = AppDataDiskFolder(str(tmp_path / "app")) extra = tmp_path / "extra" extra.mkdir() - def _download_wheel( # noqa: PLR0913 - distribution, # noqa: ARG001 - version_spec, # noqa: ARG001 - for_py_version, # noqa: ARG001 - search_dirs, # noqa: ARG001 - app_data, # noqa: ARG001 - to_folder, # noqa: ARG001 - env, # noqa: ARG001 - ): + def _download_wheel(**_kwargs): return wheel.path download_wheel = mocker.patch("virtualenv.seed.wheels.acquire.download_wheel", side_effect=_download_wheel) @@ -501,13 +635,13 @@ def _download_wheel( # noqa: PLR0913 } -def test_new_version_eq(): +def test_new_version_eq() -> None: now = datetime.now(tz=timezone.utc) value = NewVersion("a", now, now, "periodic") assert value == NewVersion("a", now, now, "periodic") -def test_new_version_ne(): +def test_new_version_ne() -> None: assert NewVersion("a", datetime.now(tz=timezone.utc), datetime.now(tz=timezone.utc), "periodic") != NewVersion( "a", datetime.now(tz=timezone.utc), @@ -516,7 +650,9 @@ def test_new_version_ne(): ) -def test_get_release_unsecure(mocker, caplog): +def test_get_release_unsecure(mocker, caplog, monkeypatch) -> None: + monkeypatch.setenv("VIRTUALENV_PERIODIC_UPDATE_INSECURE", "1") + @contextmanager def _release(of, context): assert of == "https://pypi.org/pypi/pip/json" @@ -536,7 +672,18 @@ def _release(of, context): assert " failed " in caplog.text -def test_get_release_fails(mocker, caplog): +def test_get_release_verified_failure_does_not_fallback(mocker, monkeypatch) -> None: + monkeypatch.delenv("VIRTUALENV_PERIODIC_UPDATE_INSECURE", raising=False) + url_o = mocker.patch("virtualenv.seed.wheels.periodic_update.urlopen", side_effect=URLError("insecure")) + + result = release_date_for_wheel_path(Path("pip-20.1.whl")) + + assert result is None + assert url_o.call_count == 1 + assert url_o.call_args.kwargs["context"] is None + + +def test_get_release_fails(mocker, caplog) -> None: exc = RuntimeError("oh no") url_o = mocker.patch("virtualenv.seed.wheels.periodic_update.urlopen", side_effect=exc) @@ -547,6 +694,72 @@ def test_get_release_fails(mocker, caplog): assert repr(exc) in caplog.text +def _pypi_release_response(mocker, filename, digests, version="20.1"): + entry = {"filename": filename, "digests": digests} + body = json.dumps({"releases": {version: [entry]}}) + return mocker.patch("virtualenv.seed.wheels.periodic_update.urlopen", return_value=StringIO(body)) + + +def test_verify_wheel_digest_matches(tmp_path, mocker) -> None: + wheel = Wheel(tmp_path / "pip-20.1-py3-none-any.whl") + wheel.path.write_bytes(b"content") + sha256 = hashlib.sha256(b"content").hexdigest() + _pypi_release_response(mocker, wheel.name, {"sha256": sha256}) + + verify_wheel_digest(wheel) # must not raise + + +def test_verify_wheel_digest_mismatch(tmp_path, mocker) -> None: + wheel = Wheel(tmp_path / "pip-20.1-py3-none-any.whl") + wheel.path.write_bytes(b"content") + _pypi_release_response(mocker, wheel.name, {"sha256": "0" * 64}) + + with pytest.raises(RuntimeError, match=r"has sha256 .* but PyPI reports"): + verify_wheel_digest(wheel) + + +@pytest.mark.parametrize( + ("filename", "digests", "version"), + [ + pytest.param("setuptools-99.0-py3-none-any.whl", {"sha256": "0" * 64}, "20.1", id="filename_not_in_release"), + pytest.param( + "pip-20.1-py3-none-any.whl", {"md5": "d41d8cd98f00b204e9800998ecf8427e"}, "20.1", id="no_sha256_digest" + ), + pytest.param("pip-20.1-py3-none-any.whl", {"sha256": "0" * 64}, "19.0", id="version_not_in_releases"), + ], +) +@pytest.mark.usefixtures("retry_sleep") +def test_verify_wheel_digest_rejects_when_unverifiable(tmp_path, mocker, filename, digests, version) -> None: + wheel = Wheel(tmp_path / "pip-20.1-py3-none-any.whl") + wheel.path.write_bytes(b"content") + _pypi_release_response(mocker, filename, digests, version) + + with pytest.raises(UnverifiedWheelError, match=r"could not obtain its sha256 from PyPI"): + verify_wheel_digest(wheel) + + +def test_verify_wheel_digest_rejects_after_retrying_pypi_lookup(tmp_path, mocker, retry_sleep) -> None: + wheel = Wheel(tmp_path / "pip-20.1-py3-none-any.whl") + wheel.path.write_bytes(b"content") + mocker.patch("virtualenv.seed.wheels.periodic_update.urlopen", side_effect=URLError("offline")) + + with pytest.raises(UnverifiedWheelError, match=r"could not obtain its sha256 from PyPI"): + verify_wheel_digest(wheel) + + assert [entry.args for entry in retry_sleep.call_args_list] == [(1.0,), (2.0,), (4.0,)] + + +@pytest.mark.usefixtures("retry_sleep") +def test_verify_wheel_digest_recovers_on_retry(tmp_path, mocker) -> None: + wheel = Wheel(tmp_path / "pip-20.1-py3-none-any.whl") + wheel.path.write_bytes(b"content") + body = json.dumps({"releases": {"20.1": [{"filename": wheel.name, "digests": {"sha256": "0" * 64}}]}}) + mocker.patch("virtualenv.seed.wheels.periodic_update.urlopen", side_effect=[URLError("offline"), StringIO(body)]) + + with pytest.raises(RuntimeError, match=r"has sha256 .* but PyPI reports 0{64}"): + verify_wheel_digest(wheel) + + def mock_download(mocker, pip_version_remote): def download(): index = 0 @@ -558,11 +771,11 @@ def download(): do = download() return mocker.patch( "virtualenv.seed.wheels.acquire.download_wheel", - side_effect=lambda *a, **k: next(do), # noqa: ARG005 + side_effect=lambda *_a, **_k: next(do), ) -def test_download_stop_with_embed(tmp_path, mocker, time_freeze): +def test_download_stop_with_embed(tmp_path, mocker, time_freeze) -> None: time_freeze(_UP_NOW) wheel = get_embed_wheel("pip", "3.9") app_data_outer = AppDataDiskFolder(str(tmp_path / "app")) @@ -579,13 +792,13 @@ def test_download_stop_with_embed(tmp_path, mocker, time_freeze): do_update("pip", "3.9", str(wheel.path), str(app_data_outer), [], True) assert download_wheel.call_count == 3 - assert url_o.call_count == 2 + assert url_o.call_count == 1 assert read_dict.call_count == 1 assert write.call_count == 1 -def test_download_manual_stop_after_one_download(tmp_path, mocker, time_freeze): +def test_download_manual_stop_after_one_download(tmp_path, mocker, time_freeze) -> None: time_freeze(_UP_NOW) wheel = get_embed_wheel("pip", "3.9") app_data_outer = AppDataDiskFolder(str(tmp_path / "app")) @@ -602,12 +815,12 @@ def test_download_manual_stop_after_one_download(tmp_path, mocker, time_freeze): do_update("pip", "3.9", str(wheel.path), str(app_data_outer), [], False) assert download_wheel.call_count == 1 - assert url_o.call_count == 2 + assert url_o.call_count == 1 assert read_dict.call_count == 1 assert write.call_count == 1 -def test_download_manual_ignores_pre_release(tmp_path, mocker, time_freeze): +def test_download_manual_ignores_pre_release(tmp_path, mocker, time_freeze) -> None: time_freeze(_UP_NOW) wheel = get_embed_wheel("pip", "3.9") app_data_outer = AppDataDiskFolder(str(tmp_path / "app")) @@ -625,7 +838,7 @@ def test_download_manual_ignores_pre_release(tmp_path, mocker, time_freeze): do_update("pip", "3.9", str(wheel.path), str(app_data_outer), [], False) assert download_wheel.call_count == 1 - assert url_o.call_count == 2 + assert url_o.call_count == 1 assert read_dict.call_count == 1 assert write.call_count == 1 wrote_json = write.call_args[0][0] @@ -640,7 +853,7 @@ def test_download_manual_ignores_pre_release(tmp_path, mocker, time_freeze): ] -def test_download_periodic_stop_at_first_usable(tmp_path, mocker, time_freeze): +def test_download_periodic_stop_at_first_usable(tmp_path, mocker, time_freeze) -> None: time_freeze(_UP_NOW) wheel = get_embed_wheel("pip", "3.9") app_data_outer = AppDataDiskFolder(str(tmp_path / "app")) @@ -652,7 +865,7 @@ def test_download_periodic_stop_at_first_usable(tmp_path, mocker, time_freeze): rel_date_gen = iter(rel_date_remote) release_date = mocker.patch( "virtualenv.seed.wheels.periodic_update.release_date_for_wheel_path", - side_effect=lambda *a, **k: next(rel_date_gen), # noqa: ARG005 + side_effect=lambda *_a, **_k: next(rel_date_gen), ) last_update = _UP_NOW - timedelta(days=14) @@ -668,7 +881,7 @@ def test_download_periodic_stop_at_first_usable(tmp_path, mocker, time_freeze): assert write.call_count == 1 -def test_download_periodic_stop_at_first_usable_with_previous_minor(tmp_path, mocker, time_freeze): +def test_download_periodic_stop_at_first_usable_with_previous_minor(tmp_path, mocker, time_freeze) -> None: time_freeze(_UP_NOW) wheel = get_embed_wheel("pip", "3.9") app_data_outer = AppDataDiskFolder(str(tmp_path / "app")) @@ -684,7 +897,7 @@ def test_download_periodic_stop_at_first_usable_with_previous_minor(tmp_path, mo rel_date_gen = iter(rel_date_remote) release_date = mocker.patch( "virtualenv.seed.wheels.periodic_update.release_date_for_wheel_path", - side_effect=lambda *a, **k: next(rel_date_gen), # noqa: ARG005 + side_effect=lambda *_a, **_k: next(rel_date_gen), ) last_update = _UP_NOW - timedelta(days=14) diff --git a/tests/unit/seed/wheels/test_wheels_util.py b/tests/unit/seed/wheels/test_wheels_util.py index 4d4ec9b9b..f1e898f64 100644 --- a/tests/unit/seed/wheels/test_wheels_util.py +++ b/tests/unit/seed/wheels/test_wheels_util.py @@ -1,36 +1,98 @@ from __future__ import annotations +import zipfile +from typing import TYPE_CHECKING + import pytest -from virtualenv.seed.wheels.embed import MAX, get_embed_wheel -from virtualenv.seed.wheels.util import Wheel +from virtualenv.seed.wheels.embed import MAX, MIN, get_embed_wheel +from virtualenv.seed.wheels.util import Wheel, discover_wheels + +if TYPE_CHECKING: + from collections.abc import Callable + from pathlib import Path + + +@pytest.mark.parametrize( + "distribution", + [pytest.param("pip", id="pip"), pytest.param("setuptools", id="setuptools")], +) +def test_embed_wheel_below_oldest_supported_is_missing(distribution: str) -> None: + assert get_embed_wheel(distribution, "3.8") is None -def test_wheel_support_no_python_requires(mocker): - wheel = get_embed_wheel("setuptools", for_py_version=None) - zip_mock = mocker.MagicMock() - mocker.patch("virtualenv.seed.wheels.util.ZipFile", new=zip_mock) - zip_mock.return_value.__enter__.return_value.read = lambda name: b"" # noqa: ARG005 +def test_embed_wheel_oldest_supported_is_present() -> None: + assert get_embed_wheel("pip", MIN) is not None - supports = wheel.support_py("3.8") - assert supports is True +def test_embed_wheel_future_version_reuses_newest() -> None: + assert str(get_embed_wheel("pip", "3.99")) == str(get_embed_wheel("pip", MAX)) -def test_bad_as_version_tuple(): + +def test_bad_as_version_tuple() -> None: with pytest.raises(ValueError, match="bad"): Wheel.as_version_tuple("bad") -def test_wheel_not_support(): - wheel = get_embed_wheel("setuptools", MAX) - assert wheel.support_py("3.3") is False +def test_wheel_not_support() -> None: + assert get_embed_wheel("setuptools", MAX).support_py("3.3") is False -def test_wheel_repr(): +def test_wheel_repr() -> None: wheel = get_embed_wheel("setuptools", MAX) assert str(wheel.path) in repr(wheel) -def test_unknown_distribution(): - wheel = get_embed_wheel("unknown", MAX) - assert wheel is None +def test_unknown_distribution() -> None: + assert get_embed_wheel("unknown", MAX) is None + + +@pytest.mark.parametrize( + ("requires", "py_version", "expected"), + [ + pytest.param("~=3.9", "3.9", True, id="compatible-release-same-minor"), + pytest.param("~=3.9", "3.14", True, id="compatible-release-later-minor"), + pytest.param("~=3.9", "4.0", False, id="compatible-release-next-major"), + pytest.param(">=3.10,!=3.11.*", "3.11", False, id="every-clause-must-hold"), + pytest.param(">=3.10,!=3.11.*", "3.12", True, id="every-clause-holds"), + pytest.param(">= 3.9 , != 3.10.*", "3.12", True, id="whitespace"), + pytest.param(">=3.6.*", "3.9", False, id="invalid-specifier-skipped"), + pytest.param("foo", "3.9", False, id="garbage-skipped"), + ], +) +def test_support_py(pip_wheel: Callable[[str, str], Wheel], requires: str, py_version: str, expected: bool) -> None: + assert pip_wheel(f"Requires-Python: {requires}\n", "1.0").support_py(py_version) is expected + + +@pytest.mark.parametrize( + ("headers", "py_version", "expected"), + [ + pytest.param("", "3.9", True, id="no-header"), + pytest.param("requires-python: >=3.12\n", "3.9", False, id="lowercase-header"), + pytest.param("Requires-Python: >=3.9,\n <3.10\n", "3.12", False, id="folded-header"), + pytest.param("\nRequires-Python: >=3.99\n", "3.9", True, id="body-only"), + ], +) +def test_support_py_reads_header( + pip_wheel: Callable[[str, str], Wheel], headers: str, py_version: str, expected: bool +) -> None: + assert pip_wheel(headers, "1.0").support_py(py_version) is expected + + +def test_discover_wheels_skips_unsupported(tmp_path: Path, pip_wheel: Callable[[str, str], Wheel]) -> None: + pip_wheel("Requires-Python: >=3.99\n", "2.0") + pip_wheel("", "1.0") + + assert [wheel.version for wheel in discover_wheels(tmp_path, "pip", None, "3.12")] == ["1.0"] + + +@pytest.fixture +def pip_wheel(tmp_path: Path) -> Callable[[str, str], Wheel]: + def build(headers: str, version: str) -> Wheel: + with zipfile.ZipFile(path := tmp_path / f"pip-{version}-py3-none-any.whl", "w") as zip_file: + zip_file.writestr( + f"pip-{version}.dist-info/METADATA", f"Metadata-Version: 2.1\nName: pip\nVersion: {version}\n{headers}" + ) + return Wheel(path) + + return build diff --git a/tests/unit/test_conftest.py b/tests/unit/test_conftest.py new file mode 100644 index 000000000..cc6f2bc73 --- /dev/null +++ b/tests/unit/test_conftest.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +from pathlib import Path +from typing import TYPE_CHECKING + +import pytest + +if TYPE_CHECKING: + from collections.abc import Callable + +pytest_plugins = ["pytester"] + + +@pytest.fixture +def collect(pytester: pytest.Pytester, request: pytest.FixtureRequest) -> Callable[[str], pytest.RunResult]: + # definition order differs from ID order, so slicing the collection order would pick other tests + pytester.makepyfile( + test_sample="def test_c(): pass\ndef test_a(): pass\ndef test_d(): pass\ndef test_b(): pass\ndef test_e(): pass\n" + ) + + def run(shard: str) -> pytest.RunResult: + # registered under a new name, the suite conftest cannot find itself to reorder around pytest-randomly + return pytester.runpytest_inprocess( + "-p", + "no:randomly", + "--collect-only", + "-q", + shard, + plugins=[request.config.pluginmanager.getplugin(str(Path(__file__).parents[1] / "conftest.py"))], + ) + + return run + + +@pytest.mark.parametrize( + ("shard", "expected"), + [ + pytest.param("--shard=1/2", ["test_c", "test_a", "test_e"], id="first"), + pytest.param("--shard=2/2", ["test_d", "test_b"], id="second"), + pytest.param("--shard=1/1", ["test_c", "test_a", "test_d", "test_b", "test_e"], id="single"), + ], +) +def test_shard_selects_every_nth_test_id( + collect: Callable[[str], pytest.RunResult], shard: str, expected: list[str] +) -> None: + result = collect(shard) + assert [line.split("::")[1] for line in result.outlines if "::" in line] == expected + + +def test_shard_reports_deselected(collect: Callable[[str], pytest.RunResult]) -> None: + collect("--shard=2/2").stdout.fnmatch_lines(["2/5 tests collected (3 deselected) in *"]) + + +@pytest.mark.parametrize("value", ["0/2", "3/2", "a/2", "1", "1/2/3"]) +def test_shard_rejects_invalid_spec(collect: Callable[[str], pytest.RunResult], value: str) -> None: + result = collect(f"--shard={value}") + assert result.ret == pytest.ExitCode.USAGE_ERROR + result.stderr.fnmatch_lines([f"*argument --shard: expected K/N with 1 <= K <= N, got '{value}'"]) diff --git a/tests/unit/test_file_limit.py b/tests/unit/test_file_limit.py index e0ab4f826..41b33136f 100644 --- a/tests/unit/test_file_limit.py +++ b/tests/unit/test_file_limit.py @@ -3,19 +3,38 @@ import errno import os import sys +from typing import TYPE_CHECKING import pytest from virtualenv.info import IMPLEMENTATION from virtualenv.run import cli_run +if TYPE_CHECKING: + from pathlib import Path + + +def _run_cli_with_full_fds(fds: list[int], tmp_path: Path) -> None: + try: + fds.extend(os.open(os.devnull, os.O_RDONLY) for _ in range(20)) + except OSError as jit_exc: # pypy, graalpy + assert jit_exc.errno == errno.EMFILE # ruff:ignore[pytest-assert-in-except] + with pytest.raises((SystemExit, OSError, RuntimeError)) as excinfo: + cli_run([str(tmp_path / "venv")]) + exc = excinfo.value + if isinstance(exc, SystemExit): + assert exc.code != 0 + elif isinstance(exc, OSError): + assert exc.errno == errno.EMFILE + else: + msg = str(exc) + assert ("code 24" in msg) or ("errno 24" in msg) or ("EMFILE" in msg) + @pytest.mark.skipif(sys.platform == "win32", reason="resource module not available on Windows") -def test_too_many_open_files(tmp_path): - """ - Test that we get a specific error when we have too many open files. - """ - import resource # noqa: PLC0415 +def test_too_many_open_files(tmp_path) -> None: + """Test that we get a specific error when we have too many open files.""" + import resource # ruff:ignore[import-outside-top-level] soft_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE) @@ -29,26 +48,7 @@ def test_too_many_open_files(tmp_path): fds = [] try: - # JIT implementations may use more file descriptors up front, so we can run out early - try: - fds.extend(os.open(os.devnull, os.O_RDONLY) for _ in range(20)) - except OSError as jit_exc: # pypy, graalpy - assert jit_exc.errno == errno.EMFILE # noqa: PT017 - - expected_exceptions = (SystemExit, OSError, RuntimeError) - with pytest.raises(expected_exceptions) as excinfo: - cli_run([str(tmp_path / "venv")]) - - exc = excinfo.value - if isinstance(exc, SystemExit): - assert exc.code != 0 - elif isinstance(exc, OSError): - assert exc.errno == errno.EMFILE - else: - # RuntimeError wrapper path: don't assert on libc-specific strerror text. - msg = str(exc) - assert ("code 24" in msg) or ("errno 24" in msg) or ("EMFILE" in msg) - + _run_cli_with_full_fds(fds, tmp_path) finally: for fd in fds: os.close(fd) diff --git a/tests/unit/test_run.py b/tests/unit/test_run.py index b61731d57..d0176150f 100644 --- a/tests/unit/test_run.py +++ b/tests/unit/test_run.py @@ -8,7 +8,7 @@ from virtualenv.run import cli_run, session_via_cli -def test_help(capsys): +def test_help(capsys) -> None: with pytest.raises(SystemExit) as context: cli_run(args=["-h", "-vvv"]) assert context.value.code == 0 @@ -18,7 +18,7 @@ def test_help(capsys): assert out -def test_version(capsys): +def test_version(capsys) -> None: with pytest.raises(SystemExit) as context: cli_run(args=["--version"]) assert context.value.code == 0 @@ -27,13 +27,13 @@ def test_version(capsys): assert not err assert __version__ in content - import virtualenv # noqa: PLC0415 + import virtualenv # ruff:ignore[import-outside-top-level] - assert virtualenv.__file__ in content + assert virtualenv.__file__ in content.replace("\n", "") @pytest.mark.parametrize("on", [True, False]) -def test_logging_setup(caplog, on): +def test_logging_setup(caplog, on) -> None: caplog.set_level(logging.DEBUG) session_via_cli(["env"], setup_logging=on) # DEBUG only level output is generated during this phase, default output is WARN, so if on no records should be @@ -41,3 +41,15 @@ def test_logging_setup(caplog, on): assert not caplog.records else: assert caplog.records + + +def test_invalid_discovery_method_via_env(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("VIRTUALENV_DISCOVERY", "pyenv") + with pytest.raises(RuntimeError, match=r"discovery 'pyenv' is not available") as exc_info: + session_via_cli(["env"]) + + error_message = str(exc_info.value) + assert "discovery 'pyenv' is not available" in error_message + assert "Available discovery methods:" in error_message + assert "builtin" in error_message + assert "Is the plugin installed?" in error_message diff --git a/tests/unit/test_util.py b/tests/unit/test_util.py index b83c247a2..d827b6521 100644 --- a/tests/unit/test_util.py +++ b/tests/unit/test_util.py @@ -1,27 +1,130 @@ from __future__ import annotations import concurrent.futures +import os +import sys import traceback +import zipfile +from stat import S_IEXEC, S_IREAD, S_IRGRP, S_IRWXU, S_IWUSR +from typing import TYPE_CHECKING, Final import pytest +from virtualenv.app_data import _cache_dir_with_migration, _default_app_data_dir +from virtualenv.info import fs_supports_symlink +from virtualenv.util import zipapp from virtualenv.util.lock import ReentrantFileLock +from virtualenv.util.path import copy, safe_delete, symlink from virtualenv.util.subprocess import run_cmd +if TYPE_CHECKING: + from collections.abc import Callable + from pathlib import Path -def test_run_fail(tmp_path): + +def test_run_fail(tmp_path) -> None: code, out, err = run_cmd([str(tmp_path)]) assert err assert not out assert code -def test_reentrant_file_lock_is_thread_safe(tmp_path): +def test_safe_delete_removes_read_only_files(tmp_path: Path) -> None: + target = tmp_path / "target" + nested = target / "sub" + nested.mkdir(parents=True) + # the wheel image marks its files read-only via set_tree, which on Windows blocks os.unlink + marked = nested / "wheel.py" + marked.write_text("cached", encoding="utf-8") + marked.chmod(S_IREAD) + + safe_delete(target) + + assert not target.exists() + + +def test_safe_delete_missing_path_is_noop(tmp_path: Path) -> None: + safe_delete(tmp_path / "does-not-exist") # --reset-app-data before anything was cached + + +@pytest.mark.skipif(sys.platform != "win32", reason="only Windows blocks deleting an open file") +def test_safe_delete_surfaces_undeletable_entries(tmp_path: Path) -> None: + target = tmp_path / "target" + target.mkdir() + busy = target / "busy.txt" + busy.write_text("x", encoding="utf-8") + + with busy.open(encoding="utf-8"), pytest.raises(OSError, match="busy"): + safe_delete(target) + + +@pytest.mark.skipif(sys.platform == "win32", reason="Windows deletion ignores directory permissions") +def test_safe_delete_raises_the_original_error_not_a_retry_failure(tmp_path: Path) -> None: + if hasattr(os, "geteuid") and os.geteuid() == 0: + pytest.skip("root bypasses the permission bits that make the delete fail") + target = tmp_path / "target" + blocked = target / "sub" + blocked.mkdir(parents=True) + (blocked / "wheel.py").write_text("cached", encoding="utf-8") + blocked.chmod(0o000) # rmtree reports this one through os.open, which takes more than a path + + try: + with pytest.raises(PermissionError): + safe_delete(target) + finally: + blocked.chmod(S_IRWXU) + + +@pytest.mark.skipif(sys.platform == "win32", reason="Windows deletion ignores directory permissions") +def test_safe_delete_keeps_the_other_mode_bits_when_clearing_read_only(tmp_path: Path) -> None: + if hasattr(os, "geteuid") and os.geteuid() == 0: + pytest.skip("root bypasses the permission bits that make the delete fail") + target = tmp_path / "target" + target.mkdir() + blocked = target / "wheel.py" + blocked.write_text("cached", encoding="utf-8") + blocked.chmod(S_IREAD | S_IRGRP) + target.chmod(S_IREAD | S_IEXEC) # the unlink fails, so the mode the handler left behind stays observable + + try: + with pytest.raises(PermissionError): + safe_delete(target) + assert blocked.stat().st_mode & 0o777 == S_IREAD | S_IRGRP | S_IWUSR + finally: + target.chmod(S_IRWXU) + + +@pytest.mark.skipif(not fs_supports_symlink(), reason="symlink is not supported") +@pytest.mark.parametrize("operation", [pytest.param(copy, id="copy"), pytest.param(symlink, id="symlink")]) +@pytest.mark.parametrize("directory", [pytest.param(False, id="file"), pytest.param(True, id="directory")]) +@pytest.mark.parametrize("loop", [pytest.param(False, id="missing-target"), pytest.param(True, id="self-loop")]) +def test_replace_dangling_symlink( + tmp_path: Path, operation: Callable[[Path, Path], None], directory: bool, loop: bool +) -> None: + source: Final[Path] = tmp_path / "source" + if directory: + source.mkdir() + (source / "file.txt" if directory else source).write_text("new", encoding="utf-8") + destination: Final[Path] = tmp_path / "destination" + outside: Final[Path] = tmp_path / "outside" + destination.symlink_to(destination if loop else outside, target_is_directory=directory) + + operation(source, destination) + + assert ( + destination.is_symlink(), + (destination / "file.txt" if directory else destination).read_text(encoding="utf-8"), + destination.resolve(), + outside.exists(), + ) == (operation is symlink, "new", source.resolve() if operation is symlink else destination, False) + + +def test_reentrant_file_lock_is_thread_safe(tmp_path) -> None: lock = ReentrantFileLock(tmp_path) target_file = tmp_path / "target" target_file.touch() - def recreate_target_file(): + def recreate_target_file() -> None: with lock.lock_for_key("target"): target_file.unlink() target_file.touch() @@ -32,5 +135,139 @@ def recreate_target_file(): for task in tasks: try: task.result() - except Exception: # noqa: BLE001, PERF203 + except Exception: # ruff:ignore[blind-except, try-except-in-loop] pytest.fail(traceback.format_exc()) + + +class TestDefaultAppDataDir: + def test_override_env_var(self, tmp_path: Path) -> None: + custom = str(tmp_path / "custom") + env = {"VIRTUALENV_OVERRIDE_APP_DATA": custom} + assert _default_app_data_dir(env) == custom + + def test_no_override_returns_cache_dir(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("VIRTUALENV_OVERRIDE_APP_DATA", raising=False) + result = _default_app_data_dir(os.environ) + assert result + + +class TestCacheDirMigration: + def test_migrate_old_to_new(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + old_dir = str(tmp_path / "old-data") + new_dir = str(tmp_path / "new-cache") + os.makedirs(old_dir) + (tmp_path / "old-data" / "test.txt").write_text("hello", encoding="utf-8") + + monkeypatch.setattr("virtualenv.app_data.user_cache_dir", lambda **_kw: new_dir) + monkeypatch.setattr("virtualenv.app_data.user_data_dir", lambda **_kw: old_dir) + + result = _cache_dir_with_migration() + assert result == new_dir + assert os.path.isdir(new_dir) + assert not os.path.isdir(old_dir) + assert (tmp_path / "new-cache" / "test.txt").read_text(encoding="utf-8") == "hello" + + def test_no_migration_when_old_missing(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + old_dir = str(tmp_path / "old-data") + new_dir = str(tmp_path / "new-cache") + + monkeypatch.setattr("virtualenv.app_data.user_cache_dir", lambda **_kw: new_dir) + monkeypatch.setattr("virtualenv.app_data.user_data_dir", lambda **_kw: old_dir) + + result = _cache_dir_with_migration() + assert result == new_dir + assert not os.path.isdir(old_dir) + + def test_no_migration_when_new_exists(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + old_dir = str(tmp_path / "old-data") + new_dir = str(tmp_path / "new-cache") + os.makedirs(old_dir) + os.makedirs(new_dir) + (tmp_path / "old-data" / "old.txt").write_text("old", encoding="utf-8") + + monkeypatch.setattr("virtualenv.app_data.user_cache_dir", lambda **_kw: new_dir) + monkeypatch.setattr("virtualenv.app_data.user_data_dir", lambda **_kw: old_dir) + + result = _cache_dir_with_migration() + assert result == new_dir + assert os.path.isdir(old_dir) + + def test_same_dir_returns_immediately(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + same_dir = str(tmp_path / "same") + monkeypatch.setattr("virtualenv.app_data.user_cache_dir", lambda **_kw: same_dir) + monkeypatch.setattr("virtualenv.app_data.user_data_dir", lambda **_kw: same_dir) + + result = _cache_dir_with_migration() + assert result == same_dir + + def test_fallback_on_migration_error(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + old_dir = str(tmp_path / "old-data") + new_dir = str(tmp_path / "new-cache") + os.makedirs(old_dir) + + monkeypatch.setattr("virtualenv.app_data.user_cache_dir", lambda **_kw: new_dir) + monkeypatch.setattr("virtualenv.app_data.user_data_dir", lambda **_kw: old_dir) + + def broken_move(_src: str, _dst: str) -> None: + msg = "permission denied" + raise OSError(msg) + + monkeypatch.setattr("virtualenv.app_data.shutil.move", broken_move) + + result = _cache_dir_with_migration() + assert result == old_dir + + @pytest.mark.parametrize("symlink_flag", [True, False]) + def test_symlink_app_data_survives_migration( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + symlink_flag: bool, # ruff:ignore[unused-method-argument] + ) -> None: + old_dir = str(tmp_path / "old-data") + new_dir = str(tmp_path / "new-cache") + os.makedirs(old_dir) + wheel_img = tmp_path / "old-data" / "wheel" / "3.12" / "image" / "pip" + wheel_img.mkdir(parents=True) + (wheel_img / "pip.dist-info").mkdir() + (wheel_img / "pip.dist-info" / "METADATA").write_text("Name: pip", encoding="utf-8") + + venv_dir = tmp_path / "my-venv" / "lib" / "site-packages" + venv_dir.mkdir(parents=True) + try: + os.symlink(str(wheel_img / "pip.dist-info"), str(venv_dir / "pip.dist-info")) + except OSError: + pytest.skip("symlinks not supported on this filesystem") + + monkeypatch.setattr("virtualenv.app_data.user_cache_dir", lambda **_kw: new_dir) + monkeypatch.setattr("virtualenv.app_data.user_data_dir", lambda **_kw: old_dir) + + result = _cache_dir_with_migration() + assert result == new_dir + assert (tmp_path / "new-cache" / "wheel" / "3.12" / "image" / "pip" / "pip.dist-info" / "METADATA").exists() + + +@pytest.fixture +def fake_zipapp_root(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + fake_root = tmp_path / "virtualenv.pyz" + with zipfile.ZipFile(str(fake_root), "w") as zip_file: + zip_file.writestr("virtualenv/payload.txt", "hello zipapp") + monkeypatch.setattr(zipapp, "ROOT", str(fake_root)) + return fake_root + + +def test_zipapp_read_returns_payload_from_entry_inside_root(fake_zipapp_root: Path) -> None: + entry = fake_zipapp_root / "virtualenv" / "payload.txt" + assert zipapp.read(entry) == "hello zipapp" + + +@pytest.mark.parametrize( + "parts", + [ + pytest.param(("virtualenv.pyz", "..", "escape.txt"), id="escapes-via-parent"), + pytest.param(("other", "file.txt"), id="unrelated-absolute"), + ], +) +def test_zipapp_read_rejects_path_outside_root(fake_zipapp_root: Path, parts: tuple[str, ...]) -> None: + with pytest.raises(RuntimeError, match="should be within ROOT"): + zipapp.read(fake_zipapp_root.parent.joinpath(*parts)) diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 7cdebdbe7..000000000 --- a/tox.ini +++ /dev/null @@ -1,120 +0,0 @@ -[tox] -requires = - tox>=4.28 -env_list = - fix - pypy3 - 3.14 - 3.13 - 3.12 - 3.11 - 3.10 - 3.9 - 3.8 - graalpy - coverage - readme - docs - 3.14t - 3.13t -skip_missing_interpreters = true - -[testenv] -description = run tests with {basepython} -package = wheel -wheel_build_env = .pkg -extras = - test -pass_env = - CI_RUN - PYTEST_* - TERM -set_env = - COVERAGE_FILE = {toxworkdir}/.coverage.{envname} - COVERAGE_PROCESS_START = {toxinidir}/pyproject.toml - PYTHONWARNDEFAULTENCODING = 1 - _COVERAGE_SRC = {envsitepackagesdir}/virtualenv -commands = - !graalpy: coverage erase - !graalpy: coverage run -m pytest {posargs:--junitxml "{toxworkdir}/junit.{envname}.xml" tests --int} - !graalpy: coverage combine - !graalpy: coverage report --skip-covered --show-missing - !graalpy: coverage xml -o "{toxworkdir}/coverage.{envname}.xml" - !graalpy: coverage html -d {envtmpdir}/htmlcov --show-contexts --title virtualenv-{envname}-coverage - graalpy: pytest {posargs:--junitxml "{toxworkdir}/junit.{envname}.xml" tests --skip-slow} -uv_seed = true - -[testenv:fix] -description = format the code base to adhere to our styles, and complain about what we cannot do automatically -skip_install = true -deps = - pre-commit-uv>=4.1.4 -commands = - pre-commit run --all-files --show-diff-on-failure - -[testenv:readme] -description = check that the long description is valid -skip_install = true -deps = - check-wheel-contents>=0.6.2 - twine>=6.1 - uv>=0.8 -commands = - uv build --sdist --wheel --out-dir {envtmpdir} . - twine check {envtmpdir}{/}* - check-wheel-contents --no-config {envtmpdir} - -[testenv:docs] -description = build documentation -extras = - docs -commands = - sphinx-build -d "{envtmpdir}/doctree" docs "{toxworkdir}/docs_out" --color -b html {posargs:-W} - python -c 'import pathlib; print("documentation available under file://\{0\}".format(pathlib.Path(r"{toxworkdir}") / "docs_out" / "index.html"))' - -[testenv:3.14t] -base_python = {env:TOX_BASEPYTHON} - -[testenv:3.13t] -base_python = {env:TOX_BASEPYTHON} - -[testenv:upgrade] -description = upgrade pip/wheels/setuptools to latest -skip_install = true -deps = - ruff>=0.12.4 -pass_env = - UPGRADE_ADVISORY -change_dir = {toxinidir}/tasks -commands = - - python upgrade_wheels.py -uv_seed = true - -[testenv:release] -description = do a release, required posarg of the version number -deps = - gitpython>=3.1.44 - packaging>=25 - towncrier>=24.8 -change_dir = {toxinidir}/tasks -commands = - python release.py --version {posargs} - -[testenv:dev] -description = generate a DEV environment -package = editable -extras = - docs - test -commands = - uv pip tree - python -c 'import sys; print(sys.executable)' - -[testenv:zipapp] -description = generate a zipapp -skip_install = true -deps = - packaging>=25 -commands = - python tasks/make_zipapp.py -uv_seed = true diff --git a/tox.toml b/tox.toml new file mode 100644 index 000000000..a8ada1523 --- /dev/null +++ b/tox.toml @@ -0,0 +1,298 @@ +requires = [ "tox>=4.64.7" ] +env_list = [ + "3.15", + "3.14", + "3.13", + "3.12", + "3.11", + "3.10", + "3.9", + "pypy3", + "3.13t", + "3.14t", + "3.15t", + "coverage", + "docs", + "fix", + "graalpy", + "readme", + "rp", + "tasks", + "type", +] +skip_missing_interpreters = true + +[env_run_base] +description = "run tests with {env_name}" +default_base_python = [ "3.15" ] +package = "wheel" +wheel_build_env = ".pkg" +dependency_groups = [ "test" ] +pass_env = [ "CI_RUN", "PYTEST_*", "PYTHONFAULTHANDLER", "TERM" ] +set_env._COVERAGE_SRC = "{env_site_packages_dir}{/}virtualenv" +set_env.COVERAGE_FILE = "{work_dir}{/}.coverage.{env_name}" +set_env.COVERAGE_PROCESS_START = "{tox_root}{/}pyproject.toml" +set_env.PYTHONWARNDEFAULTENCODING = "1" +commands = [ + [ "coverage", "erase" ], + [ + "coverage", + "run", + "-m", + "pytest", + { replace = "posargs", default = [ + "--junitxml", + "{work_dir}{/}junit.{env_name}.xml", + "tests", + "--int", + "-n", + "auto", + "--dist", + "loadfile", + ], extend = true }, + ], + [ "coverage", "combine" ], + [ "coverage", "report", "--skip-covered", "--show-missing" ], + [ "coverage", "xml", "-o", "{work_dir}{/}coverage.{env_name}.xml" ], + [ + "coverage", + "html", + "-d", + "{env_tmp_dir}{/}htmlcov", + "--show-contexts", + "--title", + "virtualenv-{env_name}-coverage", + ], +] +uv_seed = true + +[env_pkg_base] +default_base_python = [ "3.15" ] + +[env.coverage] +description = "combine the coverage data of every test run, passed as data directories, into one report" +skip_install = true +dependency_groups = [ "coverage" ] +set_env.COVERAGE_FILE = "{work_dir}{/}.coverage" +commands = [ + [ "coverage", "combine", { replace = "posargs", extend = true } ], + [ "coverage", "report", "--skip-covered", "--show-missing" ], +] + +[env.docs] +description = "build documentation" +dependency_groups = [ "docs" ] +set_env.PYTHONUTF8 = "1" +commands = [ + [ "pre-commit", "run", "docstrfmt", "--all-files" ], + # RE2 lacks Python 3.15 wheels: https://github.com/google/re2/issues/647 + [ "uv", "tool", "run", "--python", "3.14", "--from", "proselint>=0.16", "proselint", "check", "docs" ], + [ + "sphinx-build", + "-d", + "{env_tmp_dir}{/}doctree", + "docs", + "{env:READTHEDOCS_OUTPUT:{env_tmp_dir}{/}docs_out}/html", + "--color", + "-b", + "html", + "-W", + ], + [ + "python", + "-c", + """\ + import pathlib; print(\"documentation available under file://\" + str(pathlib.Path(r\"{work_dir}\") / \"docs_out\" / \ + \"index.html\"))\ + """, + ], +] + +[env.fix] +description = "format the code base to adhere to our styles, and complain about what we cannot do automatically" +skip_install = true +dependency_groups = [ "lint" ] +commands = [ [ "pre-commit", "run", "--all-files", "--show-diff-on-failure" ] ] + +[env.graalpy] +commands = [ + [ + "python", + "-u", + "{tox_root}{/}tasks{/}run_graalpy_tests.py", + "--group", + "{env:GRAALPY_GROUP:1/1}", + "{work_dir}{/}junit.{env_name}.xml", + { replace = "posargs", default = [ "tests", "--skip-slow" ], extend = true }, + ], +] + +[env.readme] +description = "check the long description, sdist and wheel contents, and embedded SBOM" +base_python = [ "3.15" ] +skip_install = true +dependency_groups = [ "pkg-meta" ] +commands = [ + [ "uv", "build", "--sdist", "--wheel", "--out-dir", "{env_tmp_dir}", "." ], + [ "twine", "check", "{env_tmp_dir}{/}*" ], + [ "check-sdist", "--installer", "uv" ], + [ "check-wheel-contents", "--no-config", "{env_tmp_dir}" ], + [ "python", "{tox_root}{/}tasks{/}validate_sbom.py", "{env_tmp_dir}" ], +] + +[env.rp] +commands = [ + [ + "pytest", + { replace = "posargs", default = [ + "--junitxml", + "{work_dir}{/}junit.{env_name}.xml", + "tests", + "--skip-slow", + ], extend = true }, + ], +] + +[env.tasks] +description = "run the build and CI tooling tests" +base_python = [ "3.15" ] +dependency_groups = [ "tasks" ] +set_env._COVERAGE_SRC = "{tox_root}" +set_env.COVERAGE_FILE = "{work_dir}{/}.coverage.{env_name}" +set_env.COVERAGE_PROCESS_START = "{tox_root}{/}pyproject.toml" +set_env.PYTHONWARNDEFAULTENCODING = "1" +commands = [ + [ "coverage", "erase" ], + [ + "coverage", + "run", + "-m", + "pytest", + { replace = "posargs", default = [ + "--junitxml", + "{work_dir}{/}junit.{env_name}.xml", + "tests{/}tasks", + "-n", + "auto", + "--dist", + "loadfile", + ], extend = true }, + ], + [ "coverage", "combine" ], + [ + "coverage", + "report", + "--skip-covered", + "--show-missing", + "--include", + "tasks/*,tests/tasks/*", + "--fail-under", + "100", + ], +] + +[env.type] +description = "run type checker (ty) against Python 3.15" +dependency_groups = [ "type" ] +commands = [ + [ "python", "-m", "ty", "check", "src/virtualenv", "--python-version", "3.15", "--output-format", "concise" ] +] + +[env.dev] +description = "generate a DEV environment" +package = "editable" +dependency_groups = [ "dev" ] +commands = [ + [ "uv", "pip", "tree" ], + [ "python", "-c", "import sys; print(sys.executable)" ], +] + +[env.fuzz] +description = "run the atheris fuzz targets (Linux + CPython 3.12-3.14 only)" +# Atheris lacks Python 3.15 support: https://github.com/google/atheris/issues/114 +base_python = [ "3.14" ] +dependency_groups = [ "fuzz" ] +commands = [ + [ "python", "{tox_root}{/}tasks{/}fuzz_pyenv_cfg.py", "-runs=20000" ], + [ "python", "{tox_root}{/}tasks{/}fuzz_nushell_quote.py", "-runs=20000" ], + [ "python", "{tox_root}{/}tasks{/}fuzz_powershell_quote.py", "-runs=20000" ], +] + +[env.property] +description = "run the hypothesis property tests" +dependency_groups = [ "property" ] +commands = [ + [ + "pytest", + { replace = "posargs", default = [ + "--junitxml", + "{work_dir}{/}junit.{env_name}.xml", + "tests{/}property", + "-m", + "property", + ], extend = true }, + ], +] + +[env.release] +description = "do a release, required posarg of the version number" +base_python = [ "3.15" ] +skip_install = true +deps = [ "-r tasks/release-requirements.txt" ] +change_dir = "{tox_root}{/}tasks" +commands = [ [ "python", "release.py", { replace = "posargs", extend = true } ] ] + +[env.spdx] +description = "render the CycloneDX SBOM extracted from a wheel as SPDX 2.3 and validate it" +base_python = [ "3.15" ] +skip_install = true +dependency_groups = [ "sbom" ] +set_env.UV_CONSTRAINT = "{tox_root}{/}tasks{/}release-requirements.txt" +commands = [ + [ "python", "{tox_root}{/}tasks{/}cyclonedx_to_spdx.py", "virtualenv.cdx.json", "virtualenv.spdx.json" ], + [ "pyspdxtools", "--infile", "virtualenv.spdx.json" ], +] +uv_seed = false + +[env."type-3.9"] +description = "run type checker (ty) against Python 3.9" +commands = [ [ "python", "-m", "ty", "check", "src/virtualenv", "--python-version", "3.9" ] ] +base = [ "type" ] + +[env.upgrade] +description = "upgrade pip/wheels/setuptools to latest" +skip_install = true +deps = [ "pre-commit-uv>=4.2", "ruff>=0.16.10" ] +pass_env = [ "UPGRADE_ADVISORY" ] +change_dir = "{tox_root}{/}tasks" +commands = [ [ "python", "upgrade_wheels.py" ] ] +uv_seed = true + +[env.zipapp] +description = "generate a zipapp, embed its SBOM and validate the SBOM" +base_python = [ "3.15" ] +skip_install = true +deps = [ "hatchling>=1.28", "packaging>=26.3", "uv>=0.12.19" ] +dependency_groups = [ "sbom" ] +pass_env = [ "SOURCE_DATE_EPOCH" ] +set_env.PYTHONPATH = "{tox_root}" +set_env.UV_CONSTRAINT = "{tox_root}{/}tasks{/}release-requirements.txt" +commands = [ + [ + "uv", + "build", + "--wheel", + "--python", + "{env_python}", + "--build-constraints", + "{tox_root}{/}tasks{/}release-requirements.txt", + "--require-hashes", + "--out-dir", + "{env_tmp_dir}", + ".", + ], + [ "python", "tasks/make_zipapp.py", "--wheel-dir", "{env_tmp_dir}" ], + [ "python", "tasks/zipapp_sbom.py", "virtualenv.pyz", "virtualenv.pyz.cdx.json" ], + [ "python", "tasks/validate_sbom.py", "virtualenv.pyz" ], +] +uv_seed = false