ci.yaml

on:
  push:
    branches: [main, release]
  pull_request:
    types: [unlabeled, opened, synchronize, reopened]
  merge_group:
  workflow_dispatch:

name: CI

permissions:
  contents: read

# Cancel previous workflows if they are the same workflow on same ref (branch/tags)
# with the same event (push/pull_request) even they are in progress.
# This setting will help reduce the number of duplicated workflows.
concurrency:
  group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.event.pull_request.number || github.sha }}
  cancel-in-progress: true

env:
  CARGO_ARGS: --no-default-features --features stdlib,importlib,stdio,encodings,sqlite,ssl-rustls,host_env
  CARGO_ARGS_NO_SSL: --no-default-features --features stdlib,importlib,stdio,encodings,sqlite,host_env
  # Crates excluded from workspace builds:
  # - rustpython_wasm: requires wasm target
  # - rustpython-compiler-source: deprecated
  # - rustpython-venvlauncher: Windows-only
  WORKSPACE_EXCLUDES: --exclude rustpython_wasm --exclude rustpython-compiler-source --exclude rustpython-venvlauncher
  # Python version targeted by the CI.
  PYTHON_VERSION: "3.14.3"
  X86_64_PC_WINDOWS_MSVC_OPENSSL_LIB_DIR: C:\Program Files\OpenSSL\lib\VC\x64\MD
  X86_64_PC_WINDOWS_MSVC_OPENSSL_INCLUDE_DIR: C:\Program Files\OpenSSL\include
  CARGO_INCREMENTAL: 0
  CARGO_TERM_COLOR: always

jobs:
  rust_tests:
    if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
    env:
      RUST_BACKTRACE: full
    name: Run rust tests
    runs-on: ${{ matrix.os }}
    timeout-minutes: 45
    strategy:
      matrix:
        os: [macos-latest, ubuntu-latest, windows-2025]
      fail-fast: false
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          persist-credentials: false

      - uses: dtolnay/rust-toolchain@stable
        with:
          components: clippy

      - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
        with:
          save-if: ${{ github.ref == 'refs/heads/main' }}

      - name: Install macOS dependencies
        uses: ./.github/actions/install-macos-deps

      - name: run clippy
        run: cargo clippy ${{ env.CARGO_ARGS }} --workspace --all-targets ${{ env.WORKSPACE_EXCLUDES }} -- -Dwarnings

      - name: run rust tests
        run: cargo test --workspace ${{ env.WORKSPACE_EXCLUDES }} --verbose --features threading ${{ env.CARGO_ARGS }}

      - name: check compilation without threading
        run: cargo check ${{ env.CARGO_ARGS }}

      - run: cargo doc --locked
        if: runner.os == 'Linux'

      - name: check compilation without host_env (sandbox mode)
        run: |
          cargo check -p rustpython-vm --no-default-features --features compiler
          cargo check -p rustpython-stdlib --no-default-features --features compiler
          cargo build --no-default-features --features stdlib,importlib,stdio,encodings,freeze-stdlib
        if: runner.os == 'Linux'

      - name: sandbox smoke test
        run: |
          target/debug/rustpython extra_tests/snippets/sandbox_smoke.py
          target/debug/rustpython extra_tests/snippets/stdlib_re.py
        if: runner.os == 'Linux'

      - name: Test openssl build
        run: cargo build --no-default-features --features ssl-openssl
        if: runner.os == 'Linux'

      # - name: Install tk-dev for tkinter build
      #   run: sudo apt-get update && sudo apt-get install -y tk-dev
      #   if: runner.os == 'Linux'

      # - name: Test tkinter build
      #   run: cargo build --features tkinter
      #   if: runner.os == 'Linux'

      - name: Test example projects
        run: |
          cargo run --manifest-path example_projects/barebone/Cargo.toml
          cargo run --manifest-path example_projects/frozen_stdlib/Cargo.toml
        if: runner.os == 'Linux'

      - name: run update_lib tests
        run: cargo run -- -m unittest discover -s scripts/update_lib/tests -v
        env:
          PYTHONPATH: scripts
        if: runner.os == 'Linux'

  cargo_check:
    if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
    name: cargo check
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        include: 
        - os: ubuntu-latest
          target: aarch64-linux-android
        - os: ubuntu-latest
          target: i686-unknown-linux-gnu
          dependencies:
            gcc-multilib: true
        - os: ubuntu-latest
          target: i686-unknown-linux-musl
          dependencies:
            musl-tools: true
        - os: ubuntu-latest
          target: wasm32-wasip2
        - os: ubuntu-latest
          target: x86_64-unknown-freebsd
        - os: ubuntu-latest
          target: aarch64-unknown-linux-gnu
          dependencies:
            gcc-aarch64-linux-gnu: true
        - os: macos-latest
          target: aarch64-apple-ios
        - os: macos-latest
          target: x86_64-apple-darwin
      fail-fast: false
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          persist-credentials: false

      - name: Install dependencies
        uses: ./.github/actions/install-linux-deps
        # zizmor has an issue with dynamic `with`
        # with: ${{ matrix.dependencies || fromJSON('{}') }}
        with:
          gcc-multilib: ${{ matrix.dependencies.gcc-multilib || false }}
          musl-tools: ${{ matrix.dependencies.musl-tools || false }}
          gcc-aarch64-linux-gnu: ${{ matrix.dependencies.gcc-aarch64-linux-gnu || false }}

      - name: Restore cache
        uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
        if: ${{ github.ref != 'refs/heads/main' }} # Never restore on main
        with:
          path: |
            ~/.cargo/bin/
            ~/.cargo/registry/index/
            ~/.cargo/registry/cache/
            ~/.cargo/git/db/
            target/
          # key won't match, will rely on restore-keys
          key: cargo-check-${{ runner.os }}-${{ matrix.target }}
          restore-keys: |
            cargo-check-${{ runner.os }}-${{ matrix.target }}-

      - uses: dtolnay/rust-toolchain@stable
        with:
          target: ${{ matrix.target }}

      - name: Setup Android NDK
        if: ${{ matrix.target == 'aarch64-linux-android' }}
        id: setup-ndk
        uses: nttld/setup-ndk@v1
        with:
          ndk-version: r27
          add-to-path: true

      # - name: Prepare repository for redox compilation
      #   run: bash scripts/redox/uncomment-cargo.sh
      # - name: Check compilation for Redox
      #   uses: coolreader18/redoxer-action@v1
      #   with:
      #     command: check
      #     args: --ignore-rust-version

      - name: Check compilation
        run: cargo check --target "${{ matrix.target }}" ${{ env.CARGO_ARGS_NO_SSL }}
        env:
          CC_aarch64_linux_android: ${{ steps.setup-ndk.outputs.ndk-path }}/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android24-clang
          AR_aarch64_linux_android: ${{ steps.setup-ndk.outputs.ndk-path }}/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-ar
          CARGO_TARGET_AARCH64_LINUX_ANDROID_LINKER: ${{ steps.setup-ndk.outputs.ndk-path }}/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android24-clang

      - name: Save cache
        if: ${{ github.ref == 'refs/heads/main' }} # only save on main
        uses: actions/cache/save@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
        with:
          path: |
            ~/.cargo/bin/
            ~/.cargo/registry/index/
            ~/.cargo/registry/cache/
            ~/.cargo/git/db/
            target/
          key: cargo-check-${{ runner.os }}-${{ matrix.target }}-${{ hashFiles('**/Cargo.toml') }}-${{ hashFiles('Cargo.lock') }}-${{ github.sha }}

  snippets_cpython:
    if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
    env:
      RUST_BACKTRACE: full
      # Tests that can be flaky when running with multiple processes `-j 2`. We will use `-j 1` for these.
      FLAKY_MP_TESTS: >-
        test_class
        test_concurrent_futures
        test_eintr
        test_multiprocessing_fork
        test_multiprocessing_forkserver
        test_multiprocessing_spawn
    name: Run snippets and cpython tests
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        include:
        - os: macos-latest
          extra_test_args:
            - '-u all'
          env_polluting_tests: []
          skips: []
          timeout: 50
        - os: ubuntu-latest
          extra_test_args:
            - '-u all'
          env_polluting_tests: []
          skips: []
          timeout: 60
        - os: windows-2025
          extra_test_args: [] # TODO: Enable '-u all'
          env_polluting_tests: []
          skips:
            - test_rlcompleter
            - test_pathlib # panic by surrogate chars
            - test_posixpath # OSError: (22, 'The filename, directory name, or volume label syntax is incorrect. (os error 123)')
            - test_venv # couple of failing tests
          timeout: 50
      fail-fast: false
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          persist-credentials: false

      - uses: dtolnay/rust-toolchain@stable

      - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
        with:
          save-if: ${{ github.ref == 'refs/heads/main' }}

      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Install macOS dependencies
        uses: ./.github/actions/install-macos-deps
        with:
          openssl: true

      - name: build rustpython
        run: cargo build --release --verbose --features=threading,jit ${{ env.CARGO_ARGS }}

      - name: run snippets
        run: python -m pip install -r requirements.txt && pytest -v
        working-directory: ./extra_tests

      - name: Detect available cores
        id: cores
        shell: bash
        run: |
          cores=$(python -c 'print(__import__("os").process_cpu_count())')
          echo "cores=${cores}" >> "$GITHUB_OUTPUT"

      - name: Run CPython tests
        run: |
          target/release/rustpython -m test -j ${{ steps.cores.outputs.cores }} ${{ join(matrix.extra_test_args, ' ') }} --slowest --fail-env-changed --timeout 600 -v -x ${{ env.FLAKY_MP_TESTS }} ${{ join(matrix.skips, ' ') }}
        timeout-minutes: ${{ matrix.timeout }}
        env:
          RUSTPYTHON_SKIP_ENV_POLLUTERS: true

      - name: Run flaky MP CPython tests
        run: |
          target/release/rustpython -m test -j 1 ${{ join(matrix.extra_test_args, ' ') }} --slowest --fail-env-changed --timeout 600 -v ${{ env.FLAKY_MP_TESTS }}
        timeout-minutes: ${{ matrix.timeout }}
        env:
          RUSTPYTHON_SKIP_ENV_POLLUTERS: true

      - name: run cpython tests to check if env polluters have stopped polluting
        shell: bash
        run: |
          IFS=' ' read -r -a target_array <<< "$TARGETS"

          for thing in "${target_array[@]}"; do
            for i in $(seq 1 10); do
              set +e
              target/release/rustpython -m test -j 1 --slowest --fail-env-changed --timeout 600 -v "${thing}"
              exit_code=$?
              set -e
              if [ "${exit_code}" -eq 3 ]; then
                echo "Test ${thing} polluted the environment on attempt ${i}."
                break
              fi
            done
            if [ "${exit_code}" -ne 3 ]; then
              echo "Test ${thing} is no longer polluting the environment after ${i} attempts!"
              echo "Please remove ${thing} from matrix.env_polluting_tests in '.github/workflows/ci.yaml'."
              echo "Please also remove the skip decorators that include the word 'POLLUTERS' in ${thing}."
              if [ "${exit_code}" -ne 0 ]; then
                echo "Test ${thing} failed with exit code ${exit_code}."
                echo "Please investigate which test item in ${thing} is failing and either mark it as an expected failure or a skip."
              fi
              exit 1
            fi
          done
        env:
          TARGETS: ${{ join(matrix.env_polluting_tests, ' ') }}
        timeout-minutes: 15

      - if: runner.os != 'Windows'
        name: check that --install-pip succeeds
        run: |
          mkdir site-packages
          target/release/rustpython --install-pip ensurepip --user
          target/release/rustpython -m pip install six

      - name: Check that ensurepip succeeds.
        run: |
          target/release/rustpython -m ensurepip
          target/release/rustpython -c "import pip"

      - if: runner.os != 'Windows'
        name: Check if pip inside venv is functional
        run: |
          target/release/rustpython -m venv testvenv
          testvenv/bin/rustpython -m pip install wheel

      - name: Check whats_left is not broken
        shell: bash
        run: python -I scripts/whats_left.py ${{ env.CARGO_ARGS }} --features jit

  lint:
    name: Lint
    runs-on: ubuntu-latest
    permissions:
      contents: read
      checks: write
      pull-requests: write
      security-events: write # for zizmor
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          persist-credentials: false

      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - uses: dtolnay/rust-toolchain@stable
        with:
          components: rustfmt

      - uses: cargo-bins/cargo-binstall@0b24824336e2b3800b0f89d9e08b2c08bfa3dcdd # v1.17.9

      - name: cargo shear
        run: |
          cargo binstall --no-confirm cargo-shear
          cargo shear

      - name: actionlint
        uses: reviewdog/action-actionlint@0d952c597ef8459f634d7145b0b044a9699e5e43 # v1.71.0

      - name: zizmor
        uses: zizmorcore/zizmor-action@71321a20a9ded102f6e9ce5718a2fcec2c4f70d8 # v0.5.2

      - name: restore prek cache
        if: ${{ github.ref != 'refs/heads/main' }} # never restore on main
        uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
        with:
          key: prek-${{ hashFiles('.pre-commit-config.yaml') }}
          path: ~/.cache/prek

      - name: prek
        id: prek
        uses: j178/prek-action@53276d8b0d10f8b6672aa85b4588c6921d0370cc # v2.0.1
        with:
          cache: false
          show-verbose-logs: false
        continue-on-error: true

      - name: save prek cache
        if: ${{ github.ref == 'refs/heads/main' }} # only save on main
        uses: actions/cache/save@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
        with:
          key: prek-${{ hashFiles('.pre-commit-config.yaml') }}
          path: ~/.cache/prek

      - name: reviewdog
        uses: reviewdog/action-suggester@aa38384ceb608d00f84b4690cacc83a5aba307ff # 1.24.0
        with:
          level: warning
          fail_level: error
          cleanup: false

  miri:
    if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
    name: Run tests under miri
    runs-on: ubuntu-latest
    timeout-minutes: 30
    env:
      NIGHTLY_CHANNEL: nightly
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          persist-credentials: false

      - uses: dtolnay/rust-toolchain@master
        with:
          toolchain: ${{ env.NIGHTLY_CHANNEL }}
          components: miri

      - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
        with:
          save-if: ${{ github.ref == 'refs/heads/main' }}

      - name: Run tests under miri
        run: cargo +${{ env.NIGHTLY_CHANNEL }} miri test -p rustpython-vm -- miri_test
        env:
          # miri-ignore-leaks because the type-object circular reference means that there will always be
          # a memory leak, at least until we have proper cyclic gc
          MIRIFLAGS: "-Zmiri-ignore-leaks"

  wasm:
    if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
    name: Check the WASM package and demo
    runs-on: ubuntu-latest
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          persist-credentials: false

      - uses: dtolnay/rust-toolchain@stable
        with:
          components: clippy

      - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
        with:
          save-if: ${{ github.ref == 'refs/heads/main' }}

      - name: cargo clippy
        run: cargo clippy --manifest-path=crates/wasm/Cargo.toml -- -Dwarnings

      - name: install wasm-pack
        run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
      - name: install geckodriver
        run: |
          wget https://github.com/mozilla/geckodriver/releases/download/v0.36.0/geckodriver-v0.36.0-linux64.tar.gz
          mkdir geckodriver
          tar -xzf geckodriver-v0.36.0-linux64.tar.gz -C geckodriver

      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - run: python -m pip install -r requirements.txt
        working-directory: ./wasm/tests
      - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
        with:
          cache: "npm"
          cache-dependency-path: "wasm/demo/package-lock.json"
      - name: run test
        run: |
          driver_path="$(pwd)/../../geckodriver"
          export PATH="$PATH:${driver_path}"
          npm install
          npm run test
        env:
          NODE_OPTIONS: "--openssl-legacy-provider"
        working-directory: ./wasm/demo
      - uses: mwilliamson/setup-wabt-action@v3
        with: { wabt-version: "1.0.36" }
      - name: check wasm32-unknown without js
        run: |
          cd example_projects/wasm32_without_js/rustpython-without-js
          cargo build
          cd ..
          if wasm-objdump -xj Import rustpython-without-js/target/wasm32-unknown-unknown/debug/rustpython_without_js.wasm; then
            echo "ERROR: wasm32-unknown module expects imports from the host environment" >&2
          fi
          cargo run --release --manifest-path wasm-runtime/Cargo.toml rustpython-without-js/target/wasm32-unknown-unknown/debug/rustpython_without_js.wasm
      - name: build notebook demo
        if: github.ref == 'refs/heads/release'
        run: |
          npm install
          npm run dist
          mv dist ../demo/dist/notebook
        env:
          NODE_OPTIONS: "--openssl-legacy-provider"
        working-directory: ./wasm/notebook
      - name: Deploy demo to Github Pages
        if: success() && github.ref == 'refs/heads/release'
        uses: peaceiris/actions-gh-pages@v4
        env:
          ACTIONS_DEPLOY_KEY: ${{ secrets.ACTIONS_DEMO_DEPLOY_KEY }}
          PUBLISH_DIR: ./wasm/demo/dist
          EXTERNAL_REPOSITORY: RustPython/demo
          PUBLISH_BRANCH: master

  wasm-wasi:
    if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
    name: Run snippets and cpython tests on wasm-wasi
    runs-on: ubuntu-latest
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          persist-credentials: false

      - uses: dtolnay/rust-toolchain@stable
        with:
          target: wasm32-wasip1

      - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
        with:
          save-if: ${{ github.ref == 'refs/heads/main' }}

      - name: Setup Wasmer
        uses: wasmerio/setup-wasmer@v3

      - name: Install clang
        uses: ./.github/actions/install-linux-deps
        with:
          clang: true

      - name: build rustpython
        run: cargo build --release --target wasm32-wasip1 --features freeze-stdlib,stdlib --verbose
      - name: run snippets
        run: wasmer run --dir "$(pwd)" target/wasm32-wasip1/release/rustpython.wasm -- "$(pwd)/extra_tests/snippets/stdlib_random.py"
      - name: run cpython unittest
        run: wasmer run --dir "$(pwd)" target/wasm32-wasip1/release/rustpython.wasm -- "$(pwd)/Lib/test/test_int.py"